def get_repository_python_origin_from_kwargs(kwargs):
provided_repo_name = kwargs.get("repository")
if not (kwargs.get("python_file") or kwargs.get("module_name") or kwargs.get("package_name")):
raise click.UsageError("Must specify a python file or module name")
# Short-circuit the case where an attribute and no repository name is passed in,
# giving us enough information to return an origin without loading any target
# definitions - we may need to return an origin for a non-existent repository
# (e.g. to log an origin ID for an error message)
if kwargs.get("attribute") and not provided_repo_name:
if kwargs.get("python_file"):
_check_cli_arguments_none(kwargs, "module_name", "package_name")
code_pointer = CodePointer.from_python_file(
kwargs.get("python_file"),
kwargs.get("attribute"),
get_working_directory_from_kwargs(kwargs),
)
elif kwargs.get("module_name"):
_check_cli_arguments_none(kwargs, "python_file", "working_directory", "package_name")
code_pointer = CodePointer.from_module(
kwargs.get("module_name"),
kwargs.get("attribute"),
)
elif kwargs.get("package_name"):
_check_cli_arguments_none(kwargs, "python_file", "working_directory", "module_name")
code_pointer = CodePointer.from_python_package(
kwargs.get("package_name"),
kwargs.get("attribute"),
)
else:
check.failed("Must specify a Python file or module name")
return RepositoryPythonOrigin(executable_path=sys.executable, code_pointer=code_pointer)
code_pointer_dict = _get_code_pointer_dict_from_kwargs(kwargs)
if provided_repo_name is None and len(code_pointer_dict) == 1:
code_pointer = next(iter(code_pointer_dict.values()))
elif provided_repo_name is None:
raise click.UsageError(
(
"Must provide --repository as there is more than one repository. "
"Options are: {repos}."
).format(repos=_sorted_quoted(code_pointer_dict.keys()))
)
elif not provided_repo_name in code_pointer_dict:
raise click.UsageError(
'Repository "{provided_repo_name}" not found. Found {found_names} instead.'.format(
provided_repo_name=provided_repo_name,
found_names=_sorted_quoted(code_pointer_dict.keys()),
)
)
else:
code_pointer = code_pointer_dict[provided_repo_name]
return RepositoryPythonOrigin(executable_path=sys.executable, code_pointer=code_pointer)
def sync_get_external_repositories_grpc(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle,
)
repos = []
with ephemeral_grpc_api_client(
python_executable_path=repository_location_handle.executable_path
) as api_client:
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
external_repository_data = check.inst(
api_client.external_repository(
repository_python_origin=RepositoryPythonOrigin(
repository_location_handle.executable_path, pointer
)
),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def sync_get_external_repositories(repository_location_handle):
check.inst_param(
repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle,
)
repos = []
for key, pointer in repository_location_handle.repository_code_pointer_dict.items():
external_repository_data = check.inst(
execute_unary_api_cli_command(
repository_location_handle.executable_path,
'repository',
RepositoryPythonOrigin(repository_location_handle.executable_path, pointer),
),
ExternalRepositoryData,
)
repos.append(
ExternalRepository(
external_repository_data,
RepositoryHandle(
repository_name=external_repository_data.name,
repository_key=key,
repository_location_handle=repository_location_handle,
),
)
)
return repos
def get_python_origin(self):
return RepositoryPythonOrigin(
executable_path=self.executable_path if self.executable_path else sys.executable,
code_pointer=self.pointer,
container_image=self.container_image,
entry_point=self.entry_point,
)
def test_reconstruct_from_origin():
origin = PipelinePythonOrigin(
pipeline_name="foo_pipe",
repository_origin=RepositoryPythonOrigin(
executable_path="my_python",
code_pointer=FileCodePointer(
python_file="foo.py",
fn_name="bar",
working_directory="/",
),
container_image="my_image",
entry_point=DEFAULT_DAGSTER_ENTRY_POINT,
container_context={"docker": {
"registry": "my_reg"
}},
),
)
recon_pipeline = recon_pipeline_from_origin(origin)
assert recon_pipeline.pipeline_name == origin.pipeline_name
assert recon_pipeline.repository.pointer == origin.repository_origin.code_pointer
assert recon_pipeline.repository.container_image == origin.repository_origin.container_image
assert recon_pipeline.repository.executable_path == origin.repository_origin.executable_path
assert recon_pipeline.repository.container_context == origin.repository_origin.container_context
def test_queued_pipeline_origin_check():
code_pointer = ModuleCodePointer("fake", "fake")
fake_pipeline_origin = ExternalPipelineOrigin(
ExternalRepositoryOrigin(
InProcessRepositoryLocationOrigin(
ReconstructableRepository(code_pointer)),
"foo_repo",
),
"foo",
)
fake_code_origin = PipelinePythonOrigin(
pipeline_name="foo",
repository_origin=RepositoryPythonOrigin(
sys.executable,
code_pointer,
),
)
PipelineRun(
status=PipelineRunStatus.QUEUED,
external_pipeline_origin=fake_pipeline_origin,
pipeline_code_origin=fake_code_origin,
)
with pytest.raises(check.CheckError):
PipelineRun(status=PipelineRunStatus.QUEUED)
with pytest.raises(check.CheckError):
PipelineRun().with_status(PipelineRunStatus.QUEUED)
def _get_repository_python_origin(executable_path,
repository_code_pointer_dict,
repository_name):
if repository_name not in repository_code_pointer_dict:
raise DagsterInvariantViolationError(
"Unable to find repository name {} on GRPC server.".format(
repository_name))
code_pointer = repository_code_pointer_dict[repository_name]
return RepositoryPythonOrigin(executable_path=executable_path,
code_pointer=code_pointer)
def get_origin(self):
if isinstance(self.repository_location_handle,
InProcessRepositoryLocationHandle):
return RepositoryPythonOrigin(
code_pointer=self.repository_location_handle.
repository_code_pointer_dict[self.repository_key],
executable_path=sys.executable,
)
elif isinstance(self.repository_location_handle,
PythonEnvRepositoryLocationHandle):
return RepositoryPythonOrigin(
code_pointer=self.repository_location_handle.
repository_code_pointer_dict[self.repository_key],
executable_path=self.repository_location_handle.
executable_path,
)
else:
check.failed(
'Can not target represented RepositoryDefinition locally for repository from a {}.'
.format(self.repository_location_handle.__class__.__name__))
def get_repository_origin_from_kwargs(kwargs):
load_target = created_workspace_load_target(kwargs)
if isinstance(load_target, PythonFileTarget):
return RepositoryPythonOrigin(
executable_path=sys.executable,
code_pointer=CodePointer.from_python_file(
load_target.python_file,
load_target.attribute,
working_directory=load_target.working_directory,
),
)
elif isinstance(load_target, ModuleTarget):
return RepositoryPythonOrigin(
executable_path=sys.executable,
code_pointer=CodePointer.from_module(load_target.module_name,
load_target.attribute),
)
else:
check.failed('invalid')
def get_repository_python_origin(self, repository_name: str) -> "RepositoryPythonOrigin":
if repository_name not in self.repository_code_pointer_dict:
raise DagsterInvariantViolationError(
"Unable to find repository name {} on GRPC server.".format(repository_name)
)
code_pointer = self.repository_code_pointer_dict[repository_name]
return RepositoryPythonOrigin(
executable_path=self.executable_path,
code_pointer=code_pointer,
container_image=self.container_image,
)
def get_repository_python_origin(self, repository_name):
check.str_param(repository_name, "repository_name")
list_repositories_reply = self.client.list_repositories()
repository_code_pointer_dict = list_repositories_reply.repository_code_pointer_dict
if repository_name not in repository_code_pointer_dict:
raise DagsterInvariantViolationError(
"Unable to find repository name {} on GRPC server.".format(repository_name)
)
code_pointer = repository_code_pointer_dict[repository_name]
return RepositoryPythonOrigin(
executable_path=list_repositories_reply.executable_path or sys.executable,
code_pointer=code_pointer,
)
def get_origin(self):
"""
Hack! Inject origin that the k8s images will use. The BK image uses a different directory
structure (/workdir/python_modules/dagster-test/dagster_test/test_project) than the images
inside the kind cluster (/dagster_test/test_project). As a result the normal origin won't
work, we need to inject this one.
"""
return PipelinePythonOrigin(
self._pipeline_index.name,
RepositoryPythonOrigin(
executable_path="python",
code_pointer=FileCodePointer(
"/dagster_test/test_project/test_pipelines/repo.py",
"define_demo_execution_repo",
),
),
)
def get_python_origin(self):
"""
Hack! Inject origin that the docker-celery images will use. The BK image uses a different
directory structure (/workdir/python_modules/dagster-test/dagster_test/test_project) than
the test that creates the ReconstructablePipeline. As a result the normal origin won't
work, we need to inject this one.
"""
return PipelinePythonOrigin(
self.pipeline_name,
RepositoryPythonOrigin(
executable_path="python",
code_pointer=FileCodePointer(
"/dagster_test/test_project/test_pipelines/repo.py",
"define_demo_execution_repo",
),
container_image=self.repository.container_image,
),
)
def get_origin(self):
return RepositoryPythonOrigin(executable_path=sys.executable, code_pointer=self.pointer)
def fake_repo_target():
return RepositoryPythonOrigin(sys.executable,
ModuleCodePointer("fake", "fake"))