def get_repository_python_origin_from_kwargs(kwargs): provided_repo_name = kwargs.get("repository") if not (kwargs.get("python_file") or kwargs.get("module_name") or kwargs.get("package_name")): raise click.UsageError("Must specify a python file or module name") # Short-circuit the case where an attribute and no repository name is passed in, # giving us enough information to return an origin without loading any target # definitions - we may need to return an origin for a non-existent repository # (e.g. to log an origin ID for an error message) if kwargs.get("attribute") and not provided_repo_name: if kwargs.get("python_file"): _check_cli_arguments_none(kwargs, "module_name", "package_name") code_pointer = CodePointer.from_python_file( kwargs.get("python_file"), kwargs.get("attribute"), get_working_directory_from_kwargs(kwargs), ) elif kwargs.get("module_name"): _check_cli_arguments_none(kwargs, "python_file", "working_directory", "package_name") code_pointer = CodePointer.from_module( kwargs.get("module_name"), kwargs.get("attribute"), ) elif kwargs.get("package_name"): _check_cli_arguments_none(kwargs, "python_file", "working_directory", "module_name") code_pointer = CodePointer.from_python_package( kwargs.get("package_name"), kwargs.get("attribute"), ) else: check.failed("Must specify a Python file or module name") return RepositoryPythonOrigin(executable_path=sys.executable, code_pointer=code_pointer) code_pointer_dict = _get_code_pointer_dict_from_kwargs(kwargs) if provided_repo_name is None and len(code_pointer_dict) == 1: code_pointer = next(iter(code_pointer_dict.values())) elif provided_repo_name is None: raise click.UsageError( ( "Must provide --repository as there is more than one repository. " "Options are: {repos}." ).format(repos=_sorted_quoted(code_pointer_dict.keys())) ) elif not provided_repo_name in code_pointer_dict: raise click.UsageError( 'Repository "{provided_repo_name}" not found. Found {found_names} instead.'.format( provided_repo_name=provided_repo_name, found_names=_sorted_quoted(code_pointer_dict.keys()), ) ) else: code_pointer = code_pointer_dict[provided_repo_name] return RepositoryPythonOrigin(executable_path=sys.executable, code_pointer=code_pointer)
def sync_get_external_repositories_grpc(repository_location_handle): check.inst_param( repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle, ) repos = [] with ephemeral_grpc_api_client( python_executable_path=repository_location_handle.executable_path ) as api_client: for key, pointer in repository_location_handle.repository_code_pointer_dict.items(): external_repository_data = check.inst( api_client.external_repository( repository_python_origin=RepositoryPythonOrigin( repository_location_handle.executable_path, pointer ) ), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_key=key, repository_location_handle=repository_location_handle, ), ) ) return repos
def sync_get_external_repositories(repository_location_handle): check.inst_param( repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle, ) repos = [] for key, pointer in repository_location_handle.repository_code_pointer_dict.items(): external_repository_data = check.inst( execute_unary_api_cli_command( repository_location_handle.executable_path, 'repository', RepositoryPythonOrigin(repository_location_handle.executable_path, pointer), ), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_key=key, repository_location_handle=repository_location_handle, ), ) ) return repos
def get_python_origin(self): return RepositoryPythonOrigin( executable_path=self.executable_path if self.executable_path else sys.executable, code_pointer=self.pointer, container_image=self.container_image, entry_point=self.entry_point, )
def test_reconstruct_from_origin(): origin = PipelinePythonOrigin( pipeline_name="foo_pipe", repository_origin=RepositoryPythonOrigin( executable_path="my_python", code_pointer=FileCodePointer( python_file="foo.py", fn_name="bar", working_directory="/", ), container_image="my_image", entry_point=DEFAULT_DAGSTER_ENTRY_POINT, container_context={"docker": { "registry": "my_reg" }}, ), ) recon_pipeline = recon_pipeline_from_origin(origin) assert recon_pipeline.pipeline_name == origin.pipeline_name assert recon_pipeline.repository.pointer == origin.repository_origin.code_pointer assert recon_pipeline.repository.container_image == origin.repository_origin.container_image assert recon_pipeline.repository.executable_path == origin.repository_origin.executable_path assert recon_pipeline.repository.container_context == origin.repository_origin.container_context
def test_queued_pipeline_origin_check(): code_pointer = ModuleCodePointer("fake", "fake") fake_pipeline_origin = ExternalPipelineOrigin( ExternalRepositoryOrigin( InProcessRepositoryLocationOrigin( ReconstructableRepository(code_pointer)), "foo_repo", ), "foo", ) fake_code_origin = PipelinePythonOrigin( pipeline_name="foo", repository_origin=RepositoryPythonOrigin( sys.executable, code_pointer, ), ) PipelineRun( status=PipelineRunStatus.QUEUED, external_pipeline_origin=fake_pipeline_origin, pipeline_code_origin=fake_code_origin, ) with pytest.raises(check.CheckError): PipelineRun(status=PipelineRunStatus.QUEUED) with pytest.raises(check.CheckError): PipelineRun().with_status(PipelineRunStatus.QUEUED)
def _get_repository_python_origin(executable_path, repository_code_pointer_dict, repository_name): if repository_name not in repository_code_pointer_dict: raise DagsterInvariantViolationError( "Unable to find repository name {} on GRPC server.".format( repository_name)) code_pointer = repository_code_pointer_dict[repository_name] return RepositoryPythonOrigin(executable_path=executable_path, code_pointer=code_pointer)
def get_origin(self): if isinstance(self.repository_location_handle, InProcessRepositoryLocationHandle): return RepositoryPythonOrigin( code_pointer=self.repository_location_handle. repository_code_pointer_dict[self.repository_key], executable_path=sys.executable, ) elif isinstance(self.repository_location_handle, PythonEnvRepositoryLocationHandle): return RepositoryPythonOrigin( code_pointer=self.repository_location_handle. repository_code_pointer_dict[self.repository_key], executable_path=self.repository_location_handle. executable_path, ) else: check.failed( 'Can not target represented RepositoryDefinition locally for repository from a {}.' .format(self.repository_location_handle.__class__.__name__))
def get_repository_origin_from_kwargs(kwargs): load_target = created_workspace_load_target(kwargs) if isinstance(load_target, PythonFileTarget): return RepositoryPythonOrigin( executable_path=sys.executable, code_pointer=CodePointer.from_python_file( load_target.python_file, load_target.attribute, working_directory=load_target.working_directory, ), ) elif isinstance(load_target, ModuleTarget): return RepositoryPythonOrigin( executable_path=sys.executable, code_pointer=CodePointer.from_module(load_target.module_name, load_target.attribute), ) else: check.failed('invalid')
def get_repository_python_origin(self, repository_name: str) -> "RepositoryPythonOrigin": if repository_name not in self.repository_code_pointer_dict: raise DagsterInvariantViolationError( "Unable to find repository name {} on GRPC server.".format(repository_name) ) code_pointer = self.repository_code_pointer_dict[repository_name] return RepositoryPythonOrigin( executable_path=self.executable_path, code_pointer=code_pointer, container_image=self.container_image, )
def get_repository_python_origin(self, repository_name): check.str_param(repository_name, "repository_name") list_repositories_reply = self.client.list_repositories() repository_code_pointer_dict = list_repositories_reply.repository_code_pointer_dict if repository_name not in repository_code_pointer_dict: raise DagsterInvariantViolationError( "Unable to find repository name {} on GRPC server.".format(repository_name) ) code_pointer = repository_code_pointer_dict[repository_name] return RepositoryPythonOrigin( executable_path=list_repositories_reply.executable_path or sys.executable, code_pointer=code_pointer, )
def get_origin(self): """ Hack! Inject origin that the k8s images will use. The BK image uses a different directory structure (/workdir/python_modules/dagster-test/dagster_test/test_project) than the images inside the kind cluster (/dagster_test/test_project). As a result the normal origin won't work, we need to inject this one. """ return PipelinePythonOrigin( self._pipeline_index.name, RepositoryPythonOrigin( executable_path="python", code_pointer=FileCodePointer( "/dagster_test/test_project/test_pipelines/repo.py", "define_demo_execution_repo", ), ), )
def get_python_origin(self): """ Hack! Inject origin that the docker-celery images will use. The BK image uses a different directory structure (/workdir/python_modules/dagster-test/dagster_test/test_project) than the test that creates the ReconstructablePipeline. As a result the normal origin won't work, we need to inject this one. """ return PipelinePythonOrigin( self.pipeline_name, RepositoryPythonOrigin( executable_path="python", code_pointer=FileCodePointer( "/dagster_test/test_project/test_pipelines/repo.py", "define_demo_execution_repo", ), container_image=self.repository.container_image, ), )
def get_origin(self): return RepositoryPythonOrigin(executable_path=sys.executable, code_pointer=self.pointer)
def fake_repo_target(): return RepositoryPythonOrigin(sys.executable, ModuleCodePointer("fake", "fake"))