def build_code_pointers_by_repo_name(loadable_target_origin, loadable_repository_symbols):
repository_code_pointer_dict = {}
for loadable_repository_symbol in loadable_repository_symbols:
if loadable_target_origin.python_file:
repository_code_pointer_dict[
loadable_repository_symbol.repository_name
] = CodePointer.from_python_file(
loadable_target_origin.python_file,
loadable_repository_symbol.attribute,
loadable_target_origin.working_directory,
)
elif loadable_target_origin.package_name:
repository_code_pointer_dict[
loadable_repository_symbol.repository_name
] = CodePointer.from_python_package(
loadable_target_origin.package_name, loadable_repository_symbol.attribute,
)
else:
repository_code_pointer_dict[
loadable_repository_symbol.repository_name
] = CodePointer.from_module(
loadable_target_origin.module_name, loadable_repository_symbol.attribute,
)
return repository_code_pointer_dict
def get_repository_python_origin_from_kwargs(kwargs):
provided_repo_name = kwargs.get("repository")
if not (kwargs.get("python_file") or kwargs.get("module_name")
or kwargs.get("package_name")):
raise click.UsageError("Must specify a python file or module name")
# Short-circuit the case where an attribute and no repository name is passed in,
# giving us enough information to return an origin without loading any target
# definitions - we may need to return an origin for a non-existent repository
# (e.g. to log an origin ID for an error message)
if kwargs.get("attribute") and not provided_repo_name:
if kwargs.get("python_file"):
_check_cli_arguments_none(kwargs, "module_name", "package_name")
code_pointer = CodePointer.from_python_file(
kwargs.get("python_file"),
kwargs.get("attribute"),
get_working_directory_from_kwargs(kwargs),
)
elif kwargs.get("module_name"):
_check_cli_arguments_none(kwargs, "python_file",
"working_directory", "package_name")
code_pointer = CodePointer.from_module(
kwargs.get("module_name"),
kwargs.get("attribute"),
)
elif kwargs.get("package_name"):
_check_cli_arguments_none(kwargs, "python_file",
"working_directory", "module_name")
code_pointer = CodePointer.from_python_package(
kwargs.get("package_name"),
kwargs.get("attribute"),
)
else:
check.failed("Must specify a Python file or module name")
return RepositoryPythonOrigin(executable_path=sys.executable,
code_pointer=code_pointer)
code_pointer_dict = _get_code_pointer_dict_from_kwargs(kwargs)
if provided_repo_name is None and len(code_pointer_dict) == 1:
code_pointer = next(iter(code_pointer_dict.values()))
elif provided_repo_name is None:
raise click.UsageError(
("Must provide --repository as there is more than one repository. "
"Options are: {repos}.").format(
repos=_sorted_quoted(code_pointer_dict.keys())))
elif not provided_repo_name in code_pointer_dict:
raise click.UsageError(
'Repository "{provided_repo_name}" not found. Found {found_names} instead.'
.format(
provided_repo_name=provided_repo_name,
found_names=_sorted_quoted(code_pointer_dict.keys()),
))
else:
code_pointer = code_pointer_dict[provided_repo_name]
return RepositoryPythonOrigin(executable_path=sys.executable,
code_pointer=code_pointer)
def test_single_repository_in_package():
loadable_targets = loadable_targets_from_python_package(
"dagster.utils.test.toys.single_repository")
assert len(loadable_targets) == 1
symbol = loadable_targets[0].attribute
assert symbol == "single_repository"
repo_def = CodePointer.from_python_package(
"dagster.utils.test.toys.single_repository", symbol).load_target()
isinstance(repo_def, RepositoryDefinition)
assert repo_def.name == "single_repository"
def create_python_env_location(
loadable_target_origin,
location_name=None,
user_process_api=UserProcessApi.GRPC,
use_python_package=False,
):
check.inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.opt_str_param(location_name, "location_name")
check.bool_param(use_python_package, "use_python_package")
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=loadable_target_origin, location_name=location_name
)
response = sync_list_repositories(
executable_path=loadable_target_origin.executable_path,
python_file=loadable_target_origin.python_file,
module_name=loadable_target_origin.module_name,
working_directory=loadable_target_origin.working_directory,
attribute=loadable_target_origin.attribute,
)
if loadable_target_origin.python_file:
repository_code_pointer_dict = {
lrs.repository_name: CodePointer.from_python_file(
loadable_target_origin.python_file,
lrs.attribute,
loadable_target_origin.working_directory,
)
for lrs in response.repository_symbols
}
elif use_python_package:
repository_code_pointer_dict = {
lrs.repository_name: CodePointer.from_python_package(
loadable_target_origin.module_name, lrs.attribute
)
for lrs in response.repository_symbols
}
else:
repository_code_pointer_dict = {
lrs.repository_name: CodePointer.from_module(
loadable_target_origin.module_name, lrs.attribute
)
for lrs in response.repository_symbols
}
return PythonEnvRepositoryLocationHandle(
location_name=location_name
if location_name
else _assign_python_env_location_name(repository_code_pointer_dict),
loadable_target_origin=loadable_target_origin,
repository_code_pointer_dict=repository_code_pointer_dict,
)
def _get_code_pointer_dict_from_kwargs(
kwargs: Dict[str, str]) -> Dict[str, CodePointer]:
python_file = kwargs.get("python_file")
module_name = kwargs.get("module_name")
package_name = kwargs.get("package_name")
working_directory = get_working_directory_from_kwargs(kwargs)
attribute = kwargs.get("attribute")
if python_file:
_check_cli_arguments_none(kwargs, "module_name", "package_name")
return {
cast(
RepositoryDefinition,
repository_def_from_target_def(
loadable_target.target_definition),
).name: CodePointer.from_python_file(python_file,
loadable_target.attribute,
working_directory)
for loadable_target in get_loadable_targets(
python_file, module_name, package_name, working_directory,
attribute)
}
elif module_name:
_check_cli_arguments_none(kwargs, "python_file", "package_name")
return {
cast(
RepositoryDefinition,
repository_def_from_target_def(
loadable_target.target_definition),
).name:
CodePointer.from_module(module_name, loadable_target.attribute,
working_directory)
for loadable_target in get_loadable_targets(
python_file, module_name, package_name, working_directory,
attribute)
}
elif package_name:
_check_cli_arguments_none(kwargs, "module_name", "python_file")
return {
cast(
RepositoryDefinition,
repository_def_from_target_def(
loadable_target.target_definition),
).name: CodePointer.from_python_package(package_name,
loadable_target.attribute,
working_directory)
for loadable_target in get_loadable_targets(
python_file, module_name, package_name, working_directory,
attribute)
}
else:
check.failed("Must specify a Python file or module name")
def _get_code_pointer(loadable_target_origin, loadable_repository_symbol):
if loadable_target_origin.python_file:
return CodePointer.from_python_file(
loadable_target_origin.python_file,
loadable_repository_symbol.attribute,
loadable_target_origin.working_directory,
)
elif loadable_target_origin.package_name:
return CodePointer.from_python_package(
loadable_target_origin.package_name,
loadable_repository_symbol.attribute,
loadable_target_origin.working_directory,
)
else:
return CodePointer.from_module(
loadable_target_origin.module_name,
loadable_repository_symbol.attribute,
loadable_target_origin.working_directory,
)
def location_handle_from_package_name(package_name,
attribute,
user_process_api,
location_name=None):
check.str_param(package_name, 'package_name')
check.opt_str_param(attribute, 'attribute')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
check.opt_str_param(location_name, 'location_name')
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=None,
module_name=package_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
loadable_targets = ([
LoadableTarget(attribute, load_def_in_package(package_name, attribute))
] if attribute else loadable_targets_from_python_package(package_name))
repository_code_pointer_dict = {}
for loadable_target in loadable_targets:
repository_code_pointer_dict[loadable_target.target_definition.
name] = CodePointer.from_python_package(
package_name,
loadable_target.attribute)
return RepositoryLocationHandle.create_out_of_process_location(
repository_code_pointer_dict=repository_code_pointer_dict,
# default to the name of the repository symbol for now
location_name=assign_location_name(location_name,
repository_code_pointer_dict),
)
def location_handle_from_package_name(package_name,
attribute,
user_process_api,
location_name=None,
executable_path=sys.executable):
check.str_param(package_name, 'package_name')
check.opt_str_param(attribute, 'attribute')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
check.opt_str_param(location_name, 'location_name')
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
else:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
attribute=attribute,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_package(package_name, lrs.attribute)
for lrs in response.repository_symbols
},
)
def location_handle_from_package_name(package_name,
attribute,
location_name=None):
check.str_param(package_name, 'package_name')
check.opt_str_param(attribute, 'attribute')
check.opt_str_param(location_name, 'location_name')
loadable_targets = ([
LoadableTarget(attribute, load_def_in_package(package_name, attribute))
] if attribute else loadable_targets_from_python_package(package_name))
repository_code_pointer_dict = {}
for loadable_target in loadable_targets:
repository_code_pointer_dict[loadable_target.target_definition.
name] = CodePointer.from_python_package(
package_name,
loadable_target.attribute)
return RepositoryLocationHandle.create_out_of_process_location(
repository_code_pointer_dict=repository_code_pointer_dict,
# default to the name of the repository symbol for now
location_name=assign_location_name(location_name,
repository_code_pointer_dict),
)
def load_def_in_package(package_name, attribute):
return def_from_pointer(CodePointer.from_python_package(package_name, attribute))
def load_def_in_package(package_name, attribute, working_directory):
return def_from_pointer(
CodePointer.from_python_package(package_name, attribute, working_directory)
)
def _location_handle_from_python_environment_config(python_environment_config,
yaml_path,
user_process_api):
check.dict_param(python_environment_config, 'python_environment_config')
check.str_param(yaml_path, 'yaml_path')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
executable_path, target_config = (
# do shell expansion on path
os.path.expanduser(python_environment_config['executable_path']),
python_environment_config['target'],
)
check.invariant(is_target_config(target_config))
python_file_config, python_module_config, python_package_config = (
target_config.get('python_file'),
target_config.get('python_module'),
target_config.get('python_package'),
)
if python_file_config:
absolute_path, attribute, location_name, working_directory = _get_python_file_config_data(
python_file_config, yaml_path)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=absolute_path,
module_name=None,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=absolute_path,
module_name=None,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_file(absolute_path, lrs.attribute,
working_directory)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute:
CodePointer.from_python_file(absolute_path, attribute,
working_directory)
},
)
elif python_module_config:
check.invariant(python_module_config)
module_name, attribute, location_name = _get_module_config_data(
python_module_config)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=module_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=module_name,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_module(module_name, lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute: CodePointer.from_module(module_name, attribute)
},
)
else:
check.invariant(python_package_config)
package_name, attribute, location_name = _get_package_config_data(
python_package_config)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_package(package_name,
lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute:
CodePointer.from_python_package(package_name, attribute)
},
)
