def _mgr_fn(recon_repo):
'''Goes out of process but same process as host process'''
check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository)
empty_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, 'empty_repo.yaml')
)
yield [
PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='test',
repository_code_pointer_dict={
recon_repo.get_definition().name: recon_repo.pointer
},
)
),
InProcessRepositoryLocation(empty_repo),
PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='empty_repo',
repository_code_pointer_dict={
empty_repo.get_definition().name: empty_repo.pointer
},
)
),
]
def test_dagster_out_of_process_location():
env = PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_python_env_location(
location_name="test_location",
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(__file__, "setup.py"),
attribute="test_repo",
),
))
assert env.get_repository("test_repo")
def test_dagster_out_of_process_location():
env = PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='test_location',
repository_code_pointer_dict={
'test_repo':
FileCodePointer(file_relative_path(__file__, 'setup.py'),
'test_repo'),
},
))
assert env.get_repository('test_repo')
def create_app_from_workspace(workspace, instance):
check.inst_param(workspace, 'workspace', Workspace)
check.inst_param(instance, 'instance', DagsterInstance)
warn_if_compute_logs_disabled()
print('Loading repository...')
locations = []
for repository_location_handle in workspace.repository_location_handles:
if isinstance(repository_location_handle,
InProcessRepositoryLocationHandle):
# will need to change for multi repo
check.invariant(
len(repository_location_handle.repository_code_pointer_dict) ==
1)
pointer = next(
iter(repository_location_handle.repository_code_pointer_dict.
values()))
recon_repo = ReconstructableRepository(pointer)
locations.append(InProcessRepositoryLocation(recon_repo))
elif isinstance(repository_location_handle,
PythonEnvRepositoryLocationHandle):
locations.append(
PythonEnvRepositoryLocation(repository_location_handle))
else:
check.failed('{} unsupported'.format(repository_location_handle))
context = DagsterGraphQLContext(instance=instance,
locations=locations,
version=__version__)
return instantiate_app_with_views(context)
def test_origin_id():
host_location = PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable, python_file=__file__, attribute="the_repo"
),
location_name="the_location",
)
)
external_pipeline = host_location.get_repository("the_repo").get_full_external_pipeline(
"the_pipe"
)
recon_pipeline = recon_pipeline_from_origin(external_pipeline.get_origin())
assert external_pipeline.get_origin_id() == recon_pipeline.get_origin_id()
def get_test_external_repo():
return PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='test_location',
repository_code_pointer_dict={
'test_repository': FileCodePointer(__file__,
'test_repository'),
},
)).get_repository('test_repository')
def get_test_external_repo():
return PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
attribute="test_repository",
),
location_name="test_location",
)).get_repository("test_repository")
def cli_api_repo():
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable, python_file=__file__, attribute="the_repo",
)
yield PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=loadable_target_origin, location_name="test_location",
)
).get_repository("the_repo")
def define_out_of_process_context(python_file, fn_name, instance):
check.inst_param(instance, 'instance', DagsterInstance)
return DagsterGraphQLContext(
locations=[
PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
'test_location',
{fn_name: FileCodePointer(python_file, fn_name)}))
],
instance=instance,
)
def _mgr_fn(recon_repo):
'''Goes out of process but same process as host process'''
check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository)
# this is "ok" because we know the test host process containers the user code
repo_name = recon_repo.get_definition().name
yield [
PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='test',
repository_code_pointer_dict={repo_name: recon_repo.pointer},
)
)
]
def create_app_from_workspace(workspace, instance, path_prefix=''):
check.inst_param(workspace, 'workspace', Workspace)
check.inst_param(instance, 'instance', DagsterInstance)
check.str_param(path_prefix, 'path_prefix')
if path_prefix:
if not path_prefix.startswith('/'):
raise Exception('The path prefix should begin with a leading "/".')
if path_prefix.endswith('/'):
raise Exception(
'The path prefix should not include a trailing "/".')
warn_if_compute_logs_disabled()
print('Loading repository...') # pylint: disable=print-call
locations = []
for repository_location_handle in workspace.repository_location_handles:
if isinstance(repository_location_handle,
InProcessRepositoryLocationHandle):
# will need to change for multi repo
check.invariant(
len(repository_location_handle.repository_code_pointer_dict) ==
1)
pointer = next(
iter(repository_location_handle.repository_code_pointer_dict.
values()))
recon_repo = ReconstructableRepository(pointer)
locations.append(InProcessRepositoryLocation(recon_repo))
elif isinstance(repository_location_handle,
PythonEnvRepositoryLocationHandle):
locations.append(
PythonEnvRepositoryLocation(repository_location_handle))
elif isinstance(repository_location_handle,
GrpcServerRepositoryLocationHandle):
locations.append(
GrpcServerRepositoryLocation(repository_location_handle))
else:
check.failed('{} unsupported'.format(repository_location_handle))
context = DagsterGraphQLContext(instance=instance,
locations=locations,
version=__version__)
return instantiate_app_with_views(context, path_prefix)
