def sync_get_external_repositories(repository_location_handle): check.inst_param( repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle, ) repos = [] for key, pointer in repository_location_handle.repository_code_pointer_dict.items(): external_repository_data = check.inst( execute_unary_api_cli_command( repository_location_handle.executable_path, 'repository', RepositoryPythonOrigin(repository_location_handle.executable_path, pointer), ), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_key=key, repository_location_handle=repository_location_handle, ), ) ) return repos
def sync_get_external_repositories_grpc(repository_location_handle): check.inst_param( repository_location_handle, 'repository_location_handle', PythonEnvRepositoryLocationHandle, ) repos = [] with ephemeral_grpc_api_client( python_executable_path=repository_location_handle.executable_path ) as api_client: for key, pointer in repository_location_handle.repository_code_pointer_dict.items(): external_repository_data = check.inst( api_client.external_repository( repository_python_origin=RepositoryPythonOrigin( repository_location_handle.executable_path, pointer ) ), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_key=key, repository_location_handle=repository_location_handle, ), ) ) return repos
def define_test_snapshot_context(): return DagsterGraphQLOutOfProcessRepositoryContext( instance=DagsterInstance.ephemeral(), execution_manager=SynchronousExecutionManager(), external_repository=ExternalRepository.from_repository_def( define_repository()), )
def get_external_repository_from_image(image): check.str_param(image, 'image') with get_temp_dir(in_directory=get_system_temp_directory()) as tmp_dir: output_file_name = "{}.json".format(uuid4()) command = 'dagster api snapshot repository'.format( output_file=os.path.join(DEFAULT_INTERNAL_VOLUME, output_file_name) ) output = run_serialized_container_command( image=image, command=command, volumes={tmp_dir: {'bind': DEFAULT_INTERNAL_VOLUME, 'mode': DEFAULT_MODE}}, ) if len(output) != 1: print(output) raise DagsterInvariantViolationError( "Running command {command} in container {image} resulted in output of length " "{actual} lines, expected {expected} lines".format( command=command, image=image, actual=len(output), expected=1 ) ) serialized_external_repo_data = output[0] external_repo_data = deserialize_json_to_dagster_namedtuple(serialized_external_repo_data) if not isinstance(external_repo_data, ExternalRepositoryData): raise DagsterInvariantViolationError( "Deserialized snapshot is of type {received} must be a ExternalRepositoryData".format( received=type(external_repo_data) ) ) return ExternalRepository(external_repo_data)
def sync_get_external_repositories_grpc(api_client, repository_location_handle): check.inst_param( repository_location_handle, "repository_location_handle", RepositoryLocationHandle ) repos = [] for repository_name in repository_location_handle.repository_names: external_repository_data = check.inst( api_client.external_repository( external_repository_origin=ExternalRepositoryOrigin( repository_location_handle.origin, repository_name, ) ), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_location_handle=repository_location_handle, ), ) ) return repos
def sync_get_streaming_external_repositories_grpc(api_client, repository_location_handle): check.inst_param( repository_location_handle, "repository_location_handle", RepositoryLocationHandle ) repos = [] for repository_name in repository_location_handle.repository_names: external_repository_chunks = list( api_client.streaming_external_repository( external_repository_origin=ExternalRepositoryOrigin( repository_location_handle.origin, repository_name, ) ) ) external_repository_data = deserialize_json_to_dagster_namedtuple( "".join( [ chunk["serialized_external_repository_chunk"] for chunk in external_repository_chunks ] ) ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_location_handle=repository_location_handle, ), ) ) return repos
def test_create_app_with_external_repo_data(): recon_repo = ReconstructableRepository.from_yaml( file_relative_path(__file__, './repository.yaml')) assert create_app_with_external_repository( ExternalRepository.from_repository_def(recon_repo.get_definition()), DagsterInstance.ephemeral(), )
def sync_get_external_repositories_grpc(api_client, repository_location_handle): check.inst_param(repository_location_handle, 'repository_location_handle', RepositoryLocationHandle) repos = [] for repository_name in repository_location_handle.repository_names: external_repository_data = check.inst( api_client.external_repository( repository_grpc_server_origin=RepositoryGrpcServerOrigin( repository_location_handle.host, repository_location_handle.port, repository_location_handle.socket, repository_name, )), ExternalRepositoryData, ) repos.append( ExternalRepository( external_repository_data, RepositoryHandle( repository_name=external_repository_data.name, repository_location_handle=repository_location_handle, ), )) return repos
def __init__(self, recon_repo, execution_manager, reloader=None): self._recon_repo = check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) external_repo = ExternalRepository.from_repository_def( recon_repo.get_definition()) self._repositories = {external_repo.name: external_repo} self.execution_manager = check.inst_param( execution_manager, 'pipeline_execution_manager', PipelineExecutionManager) self.reloader = check.opt_inst_param(reloader, 'reloader', Reloader)
def test_container_snapshot_provider(mocker): execute_container_mock = mocker.patch( 'dagster.core.definitions.container.run_serialized_container_command', return_value=[mock_external_repository_data()], ) external_repository = get_external_repository_from_image("foo:latest") execute_container_mock.assert_called_with( image="foo:latest", command='dagster api snapshot repository', volumes=mocker.ANY, ) assert (external_repository.external_repository_data == ExternalRepository.from_repository_def( noop_repo()).external_repository_data)
def __init__(self, recon_repo, execution_manager, instance, reloader=None, version=None): super(DagsterGraphQLInProcessRepositoryContext, self).__init__( external_repository=ExternalRepository.from_repository_def( recon_repo.get_definition()), instance=instance, ) self._recon_repo = check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) self.reloader = check.opt_inst_param(reloader, 'reloader', Reloader) self.execution_manager = check.inst_param( execution_manager, 'pipeline_execution_manager', PipelineExecutionManager) self.version = version
def check_repo_and_scheduler(repository: ExternalRepository, instance: DagsterInstance) -> None: check.inst_param(repository, "repository", ExternalRepository) check.inst_param(instance, "instance", DagsterInstance) repository_name = repository.name if not repository.get_external_schedules(): raise click.UsageError( "There are no schedules defined for repository {name}.".format( name=repository_name)) if not os.getenv("DAGSTER_HOME"): raise click.UsageError(( "The environment variable $DAGSTER_HOME is not set. Dagster requires this " "environment variable to be set to an existing directory in your filesystem " "that contains your dagster instance configuration file (dagster.yaml).\n" "You can resolve this error by exporting the environment variable." "For example, you can run the following command in your shell or " "include it in your shell configuration file:\n" '\texport DAGSTER_HOME="~/dagster_home"' "\n\n"))
def sync_get_external_repositories(repository_location_handle): check.inst_param( repository_location_handle, 'repository_location_handle', RepositoryLocationHandle, ) check.param_invariant( isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle), 'repository_location_handle', ) repos = [] for key, pointer in repository_location_handle.repository_code_pointer_dict.items(): with get_temp_file_name() as output_file: parts = [ repository_location_handle.executable_path, '-m', 'dagster', 'api', 'snapshot', 'repository', output_file, ] + xplat_shlex_split(pointer.get_cli_args()) execute_command_in_subprocess(parts) external_repository_data = read_unary_response(output_file) check.inst(external_repository_data, ExternalRepositoryData) repository_handle = RepositoryHandle( repository_name=external_repository_data.name, repository_key=key, repository_location_handle=repository_location_handle, ) repos.append(ExternalRepository(external_repository_data, repository_handle)) return repos
def sync_get_external_repository(location_handle): check.inst_param(location_handle, 'location_handle', LocationHandle) with get_temp_file_name() as output_file: parts = ['dagster', 'api', 'snapshot', 'repository', output_file] + xplat_shlex_split( location_handle.pointer.get_cli_args() ) returncode = subprocess.check_call(parts) check.invariant(returncode == 0, 'dagster api cli invocation did not complete successfully') messages = list(ipc_read_event_stream(output_file)) check.invariant(len(messages) == 1) external_repository_data = messages[0] check.inst(external_repository_data, ExternalRepositoryData) return ExternalRepository( external_repository_data, RepositoryHandle(external_repository_data.name, location_handle), )
def test_out_of_process_reload_location(self, graphql_context): result = execute_dagster_graphql(graphql_context, RELOAD_REPOSITORY_LOCATION_QUERY, {"repositoryLocationName": "test"}) assert result assert result.data assert result.data["reloadRepositoryLocation"] assert result.data["reloadRepositoryLocation"][ "__typename"] == "RepositoryLocation" assert result.data["reloadRepositoryLocation"]["name"] == "test" assert result.data["reloadRepositoryLocation"]["repositories"] == [{ "name": "test_repo" }] assert result.data["reloadRepositoryLocation"][ "isReloadSupported"] is True with mock.patch( # note it where the function is *used* that needs to mocked, not # where it is defined. # see https://docs.python.org/3/library/unittest.mock.html#where-to-patch "dagster.core.host_representation.handle.sync_list_repositories_grpc" ) as cli_command_mock: with mock.patch( # note it where the function is *used* that needs to mocked, not # where it is defined. # see https://docs.python.org/3/library/unittest.mock.html#where-to-patch "dagster.core.host_representation.repository_location.sync_get_streaming_external_repositories_grpc" ) as external_repository_mock: @repository def new_repo(): return [] new_repo_data = external_repository_data_from_def(new_repo) external_repository_mock.return_value = [ ExternalRepository( new_repo_data, RepositoryHandle( "new_repo", graphql_context. repository_locations[0].location_handle), ) ] cli_command_mock.return_value = ListRepositoriesResponse( repository_symbols=[], executable_path=sys.executable, repository_code_pointer_dict={ "new_repo": CodePointer.from_python_file(__file__, "new_repo", None) }, ) result = execute_dagster_graphql( graphql_context, RELOAD_REPOSITORY_LOCATION_QUERY, {"repositoryLocationName": "test"}, ) assert cli_command_mock.call_count == 1 assert external_repository_mock.call_count == 1 assert result.data["reloadRepositoryLocation"][ "repositories"] == [{ "name": "new_repo" }]
def external_repo_from_def(repository_def, repository_handle): return ExternalRepository(external_repository_data_from_def(repository_def), repository_handle)