def get_full_external_pipeline(repo_yaml, pipeline_name):
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
return (
InProcessRepositoryLocation(recon_repo)
.get_repository("nope")
.get_full_external_pipeline(pipeline_name)
)
def legacy_get_bar_repo_handle():
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, "legacy_repository_file.yaml"))
return (RepositoryLocation.from_handle(
RepositoryLocationHandle.create_from_repository_location_origin(
InProcessRepositoryLocationOrigin(recon_repo))).get_repository(
"bar_repo").handle)
def test_schedules():
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
with open(os.path.join(temp_dir, 'dagster.yaml'), 'w') as fd:
yaml.dump(
{
'scheduler': {
'module': 'dagster.utils.test',
'class': 'FilesystemTestScheduler',
'config': {
'base_dir': temp_dir
},
}
},
fd,
default_flow_style=False,
)
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, '../repository.yaml'))
for schedule_name in [
'many_events_every_min',
'pandas_hello_world_hourly',
]:
schedule = recon_repo.get_reconstructable_schedule(
schedule_name)
result = sync_launch_scheduled_execution(schedule.get_origin())
assert isinstance(result, ScheduledExecutionSuccess)
def _mgr_fn(recon_repo):
'''Goes out of process but same process as host process'''
check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository)
empty_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, 'empty_repo.yaml')
)
yield [
PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='test',
repository_code_pointer_dict={
recon_repo.get_definition().name: recon_repo.pointer
},
)
),
InProcessRepositoryLocation(empty_repo),
PythonEnvRepositoryLocation(
RepositoryLocationHandle.create_out_of_process_location(
location_name='empty_repo',
repository_code_pointer_dict={
empty_repo.get_definition().name: empty_repo.pointer
},
)
),
]
def test_server_down():
with grpc_instance() as instance:
repo_yaml = file_relative_path(__file__, "repo.yaml")
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(
repo_yaml)
loadable_target_origin = recon_repo.get_origin().loadable_target_origin
server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin,
max_workers=4,
force_port=True)
with server_process.create_ephemeral_client() as api_client:
repository_location = GrpcServerRepositoryLocation(
RepositoryLocationHandle.create_grpc_server_location(
location_name="test",
port=api_client.port,
socket=api_client.socket,
host=api_client.host,
))
external_pipeline = repository_location.get_repository(
"nope").get_full_external_pipeline("sleepy_pipeline")
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline, run_config=None)
launcher = instance.run_launcher
launcher.launch_run(instance, pipeline_run, external_pipeline)
poll_for_step_start(instance, pipeline_run.run_id)
assert launcher.can_terminate(pipeline_run.run_id)
original_run_tags = instance.get_run_by_id(
pipeline_run.run_id).tags[GRPC_INFO_TAG]
# Replace run tags with an invalid port
instance.add_run_tags(
pipeline_run.run_id,
{
GRPC_INFO_TAG:
seven.json.dumps(
merge_dicts({"host": "localhost"},
{"port": find_free_port()}))
},
)
assert not launcher.can_terminate(pipeline_run.run_id)
instance.add_run_tags(
pipeline_run.run_id,
{
GRPC_INFO_TAG: original_run_tags,
},
)
assert launcher.terminate(pipeline_run.run_id)
server_process.wait()
def _repo_location_origins_from_config(workspace_config, yaml_path):
ensure_workspace_config(workspace_config, yaml_path)
if "repository" in workspace_config:
warnings.warn(
# link to docs once they exist
"You are using the legacy repository yaml format. Please update your file "
"to abide by the new workspace file format."
)
origin = InProcessRepositoryLocationOrigin(
ReconstructableRepository.from_legacy_repository_yaml(yaml_path)
)
return {origin.location_name: origin}
location_origins = OrderedDict()
for location_config in workspace_config["load_from"]:
origin = _location_origin_from_location_config(location_config, yaml_path)
check.invariant(
location_origins.get(origin.location_name) is None,
'Cannot have multiple locations with the same name, got multiple "{name}"'.format(
name=origin.location_name,
),
)
location_origins[origin.location_name] = origin
return location_origins
def _mgr_fn(recon_repo):
"""Goes out of process but same process as host process"""
check.inst_param(recon_repo, "recon_repo",
ReconstructableRepository)
empty_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, "empty_repo.yaml"))
with Workspace([
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(
__file__, "setup.py"),
attribute="test_repo",
),
location_name="test",
),
RepositoryLocationHandle.create_in_process_location(
empty_repo.pointer),
RepositoryLocationHandle.create_python_env_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=file_relative_path(
__file__, "setup.py"),
attribute="empty_repo",
),
location_name="empty_repo",
),
]) as workspace:
yield workspace
def define_context_for_repository_yaml(path, instance):
check.inst_param(instance, 'instance', DagsterInstance)
return DagsterGraphQLContext(
locations=[
InProcessRepositoryLocation(ReconstructableRepository.from_legacy_repository_yaml(path))
],
instance=instance,
)
def get_external_pipeline_from_in_process_location(pipeline_name):
repo_yaml = file_relative_path(__file__, 'repo.yaml')
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
yield (
InProcessRepositoryLocation(recon_repo)
.get_repository('nope')
.get_full_external_pipeline(pipeline_name)
)
def define_context_for_repository_yaml(path, instance):
check.inst_param(instance, "instance", DagsterInstance)
return DagsterGraphQLContext(
workspace=Workspace([
RepositoryLocationHandle.create_in_process_location(
ReconstructableRepository.from_legacy_repository_yaml(
path).pointer)
]),
instance=instance,
)
def get_external_pipeline_from_grpc_server_repository(pipeline_name):
repo_yaml = file_relative_path(__file__, 'repo.yaml')
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
loadable_target_origin = LoadableTargetOrigin.from_python_origin(recon_repo.get_origin())
with GrpcServerProcess(
loadable_target_origin=loadable_target_origin
).create_ephemeral_client() as server:
repository_location = GrpcServerRepositoryLocation(
RepositoryLocationHandle.create_grpc_server_location(
location_name='test', port=server.port, socket=server.socket, host='localhost',
)
)
yield repository_location.get_repository('nope').get_full_external_pipeline(pipeline_name)
def recon_repo_for_cli_args(kwargs):
"""Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations
for repo loading above.
"""
check.dict_param(kwargs, "kwargs")
_cli_load_invariant(kwargs.get("pipeline_name") is None)
if kwargs.get("workspace"):
check.not_implemented(
"Workspace not supported yet in this cli command")
if kwargs.get("repository_yaml") or all_none(kwargs):
_cli_load_invariant(kwargs.get("module_name") is None)
_cli_load_invariant(kwargs.get("python_file") is None)
_cli_load_invariant(kwargs.get("fn_name") is None)
repo_yaml = (os.path.abspath(kwargs.get("repository_yaml"))
if kwargs.get("repository_yaml") else
DEFAULT_REPOSITORY_YAML_FILENAME)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
"Verify your current working directory or CLI arguments.".format(
repo_yaml),
)
return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
elif kwargs.get("module_name") and kwargs.get("fn_name"):
_cli_load_invariant(kwargs.get("repository_yaml") is None)
_cli_load_invariant(kwargs.get("python_file") is None)
return ReconstructableRepository.for_module(kwargs["module_name"],
kwargs["fn_name"])
elif kwargs.get("python_file") and kwargs.get("fn_name"):
_cli_load_invariant(kwargs.get("repository_yaml") is None)
_cli_load_invariant(kwargs.get("module_name") is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs["python_file"]),
kwargs["fn_name"],
kwargs.get("working_directory")
if kwargs.get("working_directory") else os.getcwd(),
)
else:
_cli_load_invariant(False)
def recon_repo_for_cli_args(kwargs):
'''Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations
for repo loading above.
'''
check.dict_param(kwargs, 'kwargs')
_cli_load_invariant(kwargs.get('pipeline_name') is None)
if kwargs.get('workspace'):
check.not_implemented('Workspace not supported yet in this cli command')
if kwargs.get('repository_yaml') or all_none(kwargs):
_cli_load_invariant(kwargs.get('module_name') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
_cli_load_invariant(kwargs.get('fn_name') is None)
repo_yaml = (
os.path.abspath(kwargs.get('repository_yaml'))
if kwargs.get('repository_yaml')
else DEFAULT_REPOSITORY_YAML_FILENAME
)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
'Verify your current working directory or CLI arguments.'.format(repo_yaml),
)
return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml)
elif kwargs.get('module_name') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructableRepository.for_module(kwargs['module_name'], kwargs['fn_name'])
elif kwargs.get('python_file') and kwargs.get('fn_name'):
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs['python_file']),
kwargs['fn_name'],
kwargs.get('working_directory') if kwargs.get('working_directory') else os.getcwd(),
)
else:
_cli_load_invariant(False)
def get_external_pipeline_from_grpc_server_repository(pipeline_name):
repo_yaml = file_relative_path(__file__, "repo.yaml")
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(
repo_yaml)
loadable_target_origin = recon_repo.get_origin().loadable_target_origin
server_process = GrpcServerProcess(
loadable_target_origin=loadable_target_origin)
try:
with server_process.create_ephemeral_client() as api_client:
repository_location = GrpcServerRepositoryLocation(
RepositoryLocationHandle.create_grpc_server_location(
location_name="test",
port=api_client.port,
socket=api_client.socket,
host=api_client.host,
))
yield repository_location.get_repository(
"nope").get_full_external_pipeline(pipeline_name)
finally:
server_process.wait()
def get_main_external_repo():
return InProcessRepositoryLocation(
ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__,
'repo.yaml')), ).get_repository('test_repo')
def test_repo_construction():
repo_yaml = file_relative_path(__file__, "repo.yaml")
assert ReconstructableRepository.from_legacy_repository_yaml(
repo_yaml).get_definition()
def legacy_get_bar_repo_handle():
recon_repo = ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, 'repository_file.yaml'))
return InProcessRepositoryLocation(recon_repo).get_repository(
'bar_repo').handle
def get_main_recon_repo():
return ReconstructableRepository.from_legacy_repository_yaml(
file_relative_path(__file__, 'repo.yaml')
)
def recon_pipeline_for_cli_args(kwargs):
'''Builds a ReconstructablePipeline for CLI arguments, which can be
any of the combinations for repo/pipeline loading above.
'''
check.dict_param(kwargs, 'kwargs')
pipeline_name = kwargs.get('pipeline_name')
if pipeline_name and not isinstance(pipeline_name, six.string_types):
if len(pipeline_name) == 1:
pipeline_name = pipeline_name[0]
else:
check.failed(
'Can only handle zero or one pipeline args. Got {pipeline_name}'
.format(pipeline_name=repr(pipeline_name)))
# Pipeline from repository YAML and pipeline_name
if (pipeline_name and kwargs.get('module_name') is None
and kwargs.get('python_file') is None
and kwargs.get('repository_yaml') is not None):
_cli_load_invariant(kwargs.get('fn_name') is None)
repo_yaml = (os.path.abspath(kwargs.get('repository_yaml'))
if kwargs.get('repository_yaml') else
DEFAULT_REPOSITORY_YAML_FILENAME)
_cli_load_invariant(
os.path.exists(repo_yaml),
'Expected to use file "{}" to load repository but it does not exist. '
'Verify your current working directory or CLI arguments.'.format(
repo_yaml),
)
return ReconstructableRepository.from_legacy_repository_yaml(
repo_yaml).get_reconstructable_pipeline(pipeline_name)
# Pipeline from repository python file
elif kwargs.get('python_file') and kwargs.get('fn_name') and pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructableRepository.for_file(
os.path.abspath(kwargs['python_file']),
kwargs['fn_name']).get_reconstructable_pipeline(pipeline_name)
# Pipeline from repository module
elif kwargs.get('module_name') and kwargs.get('fn_name') and pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructableRepository.for_module(
kwargs['module_name'],
kwargs['fn_name']).get_reconstructable_pipeline(pipeline_name)
# Pipeline from pipeline python file
elif kwargs.get('python_file') and kwargs.get(
'fn_name') and not pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('module_name') is None)
return ReconstructablePipeline.for_file(
os.path.abspath(kwargs['python_file']), kwargs['fn_name'])
# Pipeline from pipeline module
elif kwargs.get('module_name') and kwargs.get(
'fn_name') and not pipeline_name:
_cli_load_invariant(kwargs.get('repository_yaml') is None)
_cli_load_invariant(kwargs.get('python_file') is None)
return ReconstructablePipeline.for_module(kwargs['module_name'],
kwargs['fn_name'])
else:
_cli_load_invariant(False)
