def test_sync_list_python_file_with_error():
python_file = file_relative_path(__file__, "error_on_load_repo.py")
with pytest.raises(DagsterUserCodeProcessError) as e:
sync_list_repositories(
sys.executable,
python_file=python_file,
module_name=None,
working_directory=None,
attribute=None,
)
assert e.value.args[0].startswith(
"(ValueError) - ValueError: User did something bad")
assert e.value.args[0].endswith(
'raise ValueError("User did something bad")\n')
def create_python_env_location(
loadable_target_origin,
location_name=None,
user_process_api=UserProcessApi.GRPC,
use_python_package=False,
):
check.inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.opt_str_param(location_name, "location_name")
check.bool_param(use_python_package, "use_python_package")
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=loadable_target_origin, location_name=location_name
)
response = sync_list_repositories(
executable_path=loadable_target_origin.executable_path,
python_file=loadable_target_origin.python_file,
module_name=loadable_target_origin.module_name,
working_directory=loadable_target_origin.working_directory,
attribute=loadable_target_origin.attribute,
)
if loadable_target_origin.python_file:
repository_code_pointer_dict = {
lrs.repository_name: CodePointer.from_python_file(
loadable_target_origin.python_file,
lrs.attribute,
loadable_target_origin.working_directory,
)
for lrs in response.repository_symbols
}
elif use_python_package:
repository_code_pointer_dict = {
lrs.repository_name: CodePointer.from_python_package(
loadable_target_origin.module_name, lrs.attribute
)
for lrs in response.repository_symbols
}
else:
repository_code_pointer_dict = {
lrs.repository_name: CodePointer.from_module(
loadable_target_origin.module_name, lrs.attribute
)
for lrs in response.repository_symbols
}
return PythonEnvRepositoryLocationHandle(
location_name=location_name
if location_name
else _assign_python_env_location_name(repository_code_pointer_dict),
loadable_target_origin=loadable_target_origin,
repository_code_pointer_dict=repository_code_pointer_dict,
)
def test_sync_list_python_file():
python_file = file_relative_path(__file__, 'api_tests_repo.py')
loadable_repo_symbols = sync_list_repositories(
sys.executable, python_file=python_file,
module_name=None).repository_symbols
assert isinstance(loadable_repo_symbols, list)
assert len(loadable_repo_symbols) == 1
assert isinstance(loadable_repo_symbols[0], LoadableRepositorySymbol)
symbol = loadable_repo_symbols[0]
assert symbol.repository_name == 'bar_repo'
assert symbol.attribute == 'bar_repo'
def test_sync_list_python_module():
loadable_repo_symbols = sync_list_repositories(
sys.executable,
python_file=None,
module_name='dagster.utils.test.hello_world_repository',
).repository_symbols
assert isinstance(loadable_repo_symbols, list)
assert len(loadable_repo_symbols) == 1
assert isinstance(loadable_repo_symbols[0], LoadableRepositorySymbol)
symbol = loadable_repo_symbols[0]
assert symbol.repository_name == 'hello_world_repository'
assert symbol.attribute == 'hello_world_repository'
def test_sync_list_python_file_multi_repo():
python_file = file_relative_path(__file__, 'multiple_repos.py')
loadable_repo_symbols = sync_list_repositories(
sys.executable, python_file=python_file,
module_name=None).repository_symbols
assert isinstance(loadable_repo_symbols, list)
assert len(loadable_repo_symbols) == 2
assert isinstance(loadable_repo_symbols[0], LoadableRepositorySymbol)
assert isinstance(loadable_repo_symbols[1], LoadableRepositorySymbol)
by_symbol = {lrs.attribute: lrs for lrs in loadable_repo_symbols}
assert by_symbol['repo_one_symbol'].repository_name == 'repo_one'
assert by_symbol['repo_two'].repository_name == 'repo_two'
def test_sync_list_python_file_attribute_multi_repo():
python_file = file_relative_path(__file__, "multiple_repos.py")
loadable_repo_symbols = sync_list_repositories(
sys.executable,
python_file=python_file,
module_name=None,
working_directory=None,
attribute="repo_one_symbol",
).repository_symbols
assert isinstance(loadable_repo_symbols, list)
assert len(loadable_repo_symbols) == 1
assert isinstance(loadable_repo_symbols[0], LoadableRepositorySymbol)
assert loadable_repo_symbols[0].repository_name == "repo_one"
assert loadable_repo_symbols[0].attribute == "repo_one_symbol"
def location_handle_from_python_file(
python_file,
attribute,
user_process_api,
location_name=None,
working_directory=None,
executable_path=sys.executable,
):
check.str_param(python_file, 'python_file')
check.opt_str_param(attribute, 'attribute')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
check.opt_str_param(location_name, 'location_name')
check.opt_str_param(working_directory, 'working_directory')
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=python_file,
module_name=None,
working_directory=working_directory,
attribute=attribute,
),
location_name=location_name,
)
else:
response = sync_list_repositories(
executable_path=executable_path,
python_file=python_file,
module_name=None,
working_directory=working_directory,
attribute=attribute,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_file(python_file, lrs.attribute,
working_directory)
for lrs in response.repository_symbols
},
)
def _location_handle_from_python_environment_config(python_environment_config,
yaml_path,
user_process_api):
check.dict_param(python_environment_config, 'python_environment_config')
check.str_param(yaml_path, 'yaml_path')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
executable_path, target_config = (
# do shell expansion on path
os.path.expanduser(python_environment_config['executable_path']),
python_environment_config['target'],
)
check.invariant(is_target_config(target_config))
python_file_config, python_module_config, python_package_config = (
target_config.get('python_file'),
target_config.get('python_module'),
target_config.get('python_package'),
)
if python_file_config:
absolute_path, attribute, location_name, working_directory = _get_python_file_config_data(
python_file_config, yaml_path)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=absolute_path,
module_name=None,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=absolute_path,
module_name=None,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_file(absolute_path, lrs.attribute,
working_directory)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute:
CodePointer.from_python_file(absolute_path, attribute,
working_directory)
},
)
elif python_module_config:
check.invariant(python_module_config)
module_name, attribute, location_name = _get_module_config_data(
python_module_config)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=module_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=module_name,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_module(module_name, lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute: CodePointer.from_module(module_name, attribute)
},
)
else:
check.invariant(python_package_config)
package_name, attribute, location_name = _get_package_config_data(
python_package_config)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_package(package_name,
lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute:
CodePointer.from_python_package(package_name, attribute)
},
)
def _location_handle_from_python_environment_config(python_environment_config, yaml_path):
check.dict_param(python_environment_config, 'python_environment_config')
check.str_param(yaml_path, 'yaml_path')
executable_path, target_config = (
# do shell expansion on path
os.path.expanduser(python_environment_config['executable_path']),
python_environment_config['target'],
)
check.invariant(is_target_config(target_config))
python_file_config, python_module_config = (
target_config.get('python_file'),
target_config.get('python_module'),
)
if python_file_config:
absolute_path, attribute, location_name = _get_python_file_config_data(
python_file_config, yaml_path
)
if not attribute:
response = sync_list_repositories(
executable_path=executable_path, python_file=absolute_path, module_name=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.attribute: CodePointer.from_python_file(absolute_path, lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute: CodePointer.from_python_file(absolute_path, attribute)
},
)
else:
check.invariant(python_module_config)
module_name, attribute, location_name = _get_module_config_data(python_module_config)
if not attribute:
response = sync_list_repositories(
executable_path=executable_path, python_file=None, module_name=module_name,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.attribute: CodePointer.from_module(module_name, lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute: CodePointer.from_module(module_name, attribute)
},
)
