def _mgr_fn(recon_repo): '''Goes out of process but same process as host process''' check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) empty_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, 'empty_repo.yaml') ) yield [ PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='test', repository_code_pointer_dict={ recon_repo.get_definition().name: recon_repo.pointer }, ) ), InProcessRepositoryLocation(empty_repo), PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='empty_repo', repository_code_pointer_dict={ empty_repo.get_definition().name: empty_repo.pointer }, ) ), ]
def create_app_from_workspace(workspace, instance): check.inst_param(workspace, 'workspace', Workspace) check.inst_param(instance, 'instance', DagsterInstance) warn_if_compute_logs_disabled() print('Loading repository...') locations = [] for repository_location_handle in workspace.repository_location_handles: if isinstance(repository_location_handle, InProcessRepositoryLocationHandle): # will need to change for multi repo check.invariant( len(repository_location_handle.repository_code_pointer_dict) == 1) pointer = next( iter(repository_location_handle.repository_code_pointer_dict. values())) recon_repo = ReconstructableRepository(pointer) locations.append(InProcessRepositoryLocation(recon_repo)) elif isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle): locations.append( PythonEnvRepositoryLocation(repository_location_handle)) else: check.failed('{} unsupported'.format(repository_location_handle)) context = DagsterGraphQLContext(instance=instance, locations=locations, version=__version__) return instantiate_app_with_views(context)
def get_test_external_repo(): return PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='test_location', repository_code_pointer_dict={ 'test_repository': FileCodePointer(__file__, 'test_repository'), }, )).get_repository('test_repository')
def get_test_external_repo(): return PythonEnvRepositoryLocation( RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, attribute="test_repository", ), location_name="test_location", )).get_repository("test_repository")
def cli_api_repo(): loadable_target_origin = LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, attribute="the_repo", ) yield PythonEnvRepositoryLocation( RepositoryLocationHandle.create_python_env_location( loadable_target_origin=loadable_target_origin, location_name="test_location", ) ).get_repository("the_repo")
def test_dagster_out_of_process_location(): env = PythonEnvRepositoryLocation( RepositoryLocationHandle.create_python_env_location( location_name="test_location", loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path(__file__, "setup.py"), attribute="test_repo", ), )) assert env.get_repository("test_repo")
def test_dagster_out_of_process_location(): env = PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='test_location', repository_code_pointer_dict={ 'test_repo': FileCodePointer(file_relative_path(__file__, 'setup.py'), 'test_repo'), }, )) assert env.get_repository('test_repo')
def define_out_of_process_context(python_file, fn_name, instance): check.inst_param(instance, 'instance', DagsterInstance) return DagsterGraphQLContext( locations=[ PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( 'test_location', {fn_name: FileCodePointer(python_file, fn_name)})) ], instance=instance, )
def _mgr_fn(recon_repo): '''Goes out of process but same process as host process''' check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) # this is "ok" because we know the test host process containers the user code repo_name = recon_repo.get_definition().name yield [ PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='test', repository_code_pointer_dict={repo_name: recon_repo.pointer}, ) ) ]
def test_origin_id(): host_location = PythonEnvRepositoryLocation( RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=__file__, attribute="the_repo" ), location_name="the_location", ) ) external_pipeline = host_location.get_repository("the_repo").get_full_external_pipeline( "the_pipe" ) recon_pipeline = recon_pipeline_from_origin(external_pipeline.get_origin()) assert external_pipeline.get_origin_id() == recon_pipeline.get_origin_id()
def create_app_from_workspace(workspace, instance, path_prefix=''): check.inst_param(workspace, 'workspace', Workspace) check.inst_param(instance, 'instance', DagsterInstance) check.str_param(path_prefix, 'path_prefix') if path_prefix: if not path_prefix.startswith('/'): raise Exception('The path prefix should begin with a leading "/".') if path_prefix.endswith('/'): raise Exception( 'The path prefix should not include a trailing "/".') warn_if_compute_logs_disabled() print('Loading repository...') # pylint: disable=print-call locations = [] for repository_location_handle in workspace.repository_location_handles: if isinstance(repository_location_handle, InProcessRepositoryLocationHandle): # will need to change for multi repo check.invariant( len(repository_location_handle.repository_code_pointer_dict) == 1) pointer = next( iter(repository_location_handle.repository_code_pointer_dict. values())) recon_repo = ReconstructableRepository(pointer) locations.append(InProcessRepositoryLocation(recon_repo)) elif isinstance(repository_location_handle, PythonEnvRepositoryLocationHandle): locations.append( PythonEnvRepositoryLocation(repository_location_handle)) elif isinstance(repository_location_handle, GrpcServerRepositoryLocationHandle): locations.append( GrpcServerRepositoryLocation(repository_location_handle)) else: check.failed('{} unsupported'.format(repository_location_handle)) context = DagsterGraphQLContext(instance=instance, locations=locations, version=__version__) return instantiate_app_with_views(context, path_prefix)