def get_full_external_pipeline(repo_yaml, pipeline_name): recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) return ( InProcessRepositoryLocation(recon_repo) .get_repository("nope") .get_full_external_pipeline(pipeline_name) )
def legacy_get_bar_repo_handle(): recon_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, "legacy_repository_file.yaml")) return (RepositoryLocation.from_handle( RepositoryLocationHandle.create_from_repository_location_origin( InProcessRepositoryLocationOrigin(recon_repo))).get_repository( "bar_repo").handle)
def test_schedules(): with seven.TemporaryDirectory() as temp_dir: with environ({'DAGSTER_HOME': temp_dir}): with open(os.path.join(temp_dir, 'dagster.yaml'), 'w') as fd: yaml.dump( { 'scheduler': { 'module': 'dagster.utils.test', 'class': 'FilesystemTestScheduler', 'config': { 'base_dir': temp_dir }, } }, fd, default_flow_style=False, ) recon_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, '../repository.yaml')) for schedule_name in [ 'many_events_every_min', 'pandas_hello_world_hourly', ]: schedule = recon_repo.get_reconstructable_schedule( schedule_name) result = sync_launch_scheduled_execution(schedule.get_origin()) assert isinstance(result, ScheduledExecutionSuccess)
def _mgr_fn(recon_repo): '''Goes out of process but same process as host process''' check.inst_param(recon_repo, 'recon_repo', ReconstructableRepository) empty_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, 'empty_repo.yaml') ) yield [ PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='test', repository_code_pointer_dict={ recon_repo.get_definition().name: recon_repo.pointer }, ) ), InProcessRepositoryLocation(empty_repo), PythonEnvRepositoryLocation( RepositoryLocationHandle.create_out_of_process_location( location_name='empty_repo', repository_code_pointer_dict={ empty_repo.get_definition().name: empty_repo.pointer }, ) ), ]
def test_server_down(): with grpc_instance() as instance: repo_yaml = file_relative_path(__file__, "repo.yaml") recon_repo = ReconstructableRepository.from_legacy_repository_yaml( repo_yaml) loadable_target_origin = recon_repo.get_origin().loadable_target_origin server_process = GrpcServerProcess( loadable_target_origin=loadable_target_origin, max_workers=4, force_port=True) with server_process.create_ephemeral_client() as api_client: repository_location = GrpcServerRepositoryLocation( RepositoryLocationHandle.create_grpc_server_location( location_name="test", port=api_client.port, socket=api_client.socket, host=api_client.host, )) external_pipeline = repository_location.get_repository( "nope").get_full_external_pipeline("sleepy_pipeline") pipeline_run = instance.create_run_for_pipeline( pipeline_def=sleepy_pipeline, run_config=None) launcher = instance.run_launcher launcher.launch_run(instance, pipeline_run, external_pipeline) poll_for_step_start(instance, pipeline_run.run_id) assert launcher.can_terminate(pipeline_run.run_id) original_run_tags = instance.get_run_by_id( pipeline_run.run_id).tags[GRPC_INFO_TAG] # Replace run tags with an invalid port instance.add_run_tags( pipeline_run.run_id, { GRPC_INFO_TAG: seven.json.dumps( merge_dicts({"host": "localhost"}, {"port": find_free_port()})) }, ) assert not launcher.can_terminate(pipeline_run.run_id) instance.add_run_tags( pipeline_run.run_id, { GRPC_INFO_TAG: original_run_tags, }, ) assert launcher.terminate(pipeline_run.run_id) server_process.wait()
def _repo_location_origins_from_config(workspace_config, yaml_path): ensure_workspace_config(workspace_config, yaml_path) if "repository" in workspace_config: warnings.warn( # link to docs once they exist "You are using the legacy repository yaml format. Please update your file " "to abide by the new workspace file format." ) origin = InProcessRepositoryLocationOrigin( ReconstructableRepository.from_legacy_repository_yaml(yaml_path) ) return {origin.location_name: origin} location_origins = OrderedDict() for location_config in workspace_config["load_from"]: origin = _location_origin_from_location_config(location_config, yaml_path) check.invariant( location_origins.get(origin.location_name) is None, 'Cannot have multiple locations with the same name, got multiple "{name}"'.format( name=origin.location_name, ), ) location_origins[origin.location_name] = origin return location_origins
def _mgr_fn(recon_repo): """Goes out of process but same process as host process""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) empty_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, "empty_repo.yaml")) with Workspace([ RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path( __file__, "setup.py"), attribute="test_repo", ), location_name="test", ), RepositoryLocationHandle.create_in_process_location( empty_repo.pointer), RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path( __file__, "setup.py"), attribute="empty_repo", ), location_name="empty_repo", ), ]) as workspace: yield workspace
def define_context_for_repository_yaml(path, instance): check.inst_param(instance, 'instance', DagsterInstance) return DagsterGraphQLContext( locations=[ InProcessRepositoryLocation(ReconstructableRepository.from_legacy_repository_yaml(path)) ], instance=instance, )
def get_external_pipeline_from_in_process_location(pipeline_name): repo_yaml = file_relative_path(__file__, 'repo.yaml') recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) yield ( InProcessRepositoryLocation(recon_repo) .get_repository('nope') .get_full_external_pipeline(pipeline_name) )
def define_context_for_repository_yaml(path, instance): check.inst_param(instance, "instance", DagsterInstance) return DagsterGraphQLContext( workspace=Workspace([ RepositoryLocationHandle.create_in_process_location( ReconstructableRepository.from_legacy_repository_yaml( path).pointer) ]), instance=instance, )
def get_external_pipeline_from_grpc_server_repository(pipeline_name): repo_yaml = file_relative_path(__file__, 'repo.yaml') recon_repo = ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) loadable_target_origin = LoadableTargetOrigin.from_python_origin(recon_repo.get_origin()) with GrpcServerProcess( loadable_target_origin=loadable_target_origin ).create_ephemeral_client() as server: repository_location = GrpcServerRepositoryLocation( RepositoryLocationHandle.create_grpc_server_location( location_name='test', port=server.port, socket=server.socket, host='localhost', ) ) yield repository_location.get_repository('nope').get_full_external_pipeline(pipeline_name)
def recon_repo_for_cli_args(kwargs): """Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations for repo loading above. """ check.dict_param(kwargs, "kwargs") _cli_load_invariant(kwargs.get("pipeline_name") is None) if kwargs.get("workspace"): check.not_implemented( "Workspace not supported yet in this cli command") if kwargs.get("repository_yaml") or all_none(kwargs): _cli_load_invariant(kwargs.get("module_name") is None) _cli_load_invariant(kwargs.get("python_file") is None) _cli_load_invariant(kwargs.get("fn_name") is None) repo_yaml = (os.path.abspath(kwargs.get("repository_yaml")) if kwargs.get("repository_yaml") else DEFAULT_REPOSITORY_YAML_FILENAME) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' "Verify your current working directory or CLI arguments.".format( repo_yaml), ) return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) elif kwargs.get("module_name") and kwargs.get("fn_name"): _cli_load_invariant(kwargs.get("repository_yaml") is None) _cli_load_invariant(kwargs.get("python_file") is None) return ReconstructableRepository.for_module(kwargs["module_name"], kwargs["fn_name"]) elif kwargs.get("python_file") and kwargs.get("fn_name"): _cli_load_invariant(kwargs.get("repository_yaml") is None) _cli_load_invariant(kwargs.get("module_name") is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs["python_file"]), kwargs["fn_name"], kwargs.get("working_directory") if kwargs.get("working_directory") else os.getcwd(), ) else: _cli_load_invariant(False)
def recon_repo_for_cli_args(kwargs): '''Builds a ReconstructableRepository for CLI arguments, which can be any of the combinations for repo loading above. ''' check.dict_param(kwargs, 'kwargs') _cli_load_invariant(kwargs.get('pipeline_name') is None) if kwargs.get('workspace'): check.not_implemented('Workspace not supported yet in this cli command') if kwargs.get('repository_yaml') or all_none(kwargs): _cli_load_invariant(kwargs.get('module_name') is None) _cli_load_invariant(kwargs.get('python_file') is None) _cli_load_invariant(kwargs.get('fn_name') is None) repo_yaml = ( os.path.abspath(kwargs.get('repository_yaml')) if kwargs.get('repository_yaml') else DEFAULT_REPOSITORY_YAML_FILENAME ) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' 'Verify your current working directory or CLI arguments.'.format(repo_yaml), ) return ReconstructableRepository.from_legacy_repository_yaml(repo_yaml) elif kwargs.get('module_name') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructableRepository.for_module(kwargs['module_name'], kwargs['fn_name']) elif kwargs.get('python_file') and kwargs.get('fn_name'): _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name'], kwargs.get('working_directory') if kwargs.get('working_directory') else os.getcwd(), ) else: _cli_load_invariant(False)
def get_external_pipeline_from_grpc_server_repository(pipeline_name): repo_yaml = file_relative_path(__file__, "repo.yaml") recon_repo = ReconstructableRepository.from_legacy_repository_yaml( repo_yaml) loadable_target_origin = recon_repo.get_origin().loadable_target_origin server_process = GrpcServerProcess( loadable_target_origin=loadable_target_origin) try: with server_process.create_ephemeral_client() as api_client: repository_location = GrpcServerRepositoryLocation( RepositoryLocationHandle.create_grpc_server_location( location_name="test", port=api_client.port, socket=api_client.socket, host=api_client.host, )) yield repository_location.get_repository( "nope").get_full_external_pipeline(pipeline_name) finally: server_process.wait()
def get_main_external_repo(): return InProcessRepositoryLocation( ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, 'repo.yaml')), ).get_repository('test_repo')
def test_repo_construction(): repo_yaml = file_relative_path(__file__, "repo.yaml") assert ReconstructableRepository.from_legacy_repository_yaml( repo_yaml).get_definition()
def legacy_get_bar_repo_handle(): recon_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, 'repository_file.yaml')) return InProcessRepositoryLocation(recon_repo).get_repository( 'bar_repo').handle
def get_main_recon_repo(): return ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, 'repo.yaml') )
def recon_pipeline_for_cli_args(kwargs): '''Builds a ReconstructablePipeline for CLI arguments, which can be any of the combinations for repo/pipeline loading above. ''' check.dict_param(kwargs, 'kwargs') pipeline_name = kwargs.get('pipeline_name') if pipeline_name and not isinstance(pipeline_name, six.string_types): if len(pipeline_name) == 1: pipeline_name = pipeline_name[0] else: check.failed( 'Can only handle zero or one pipeline args. Got {pipeline_name}' .format(pipeline_name=repr(pipeline_name))) # Pipeline from repository YAML and pipeline_name if (pipeline_name and kwargs.get('module_name') is None and kwargs.get('python_file') is None and kwargs.get('repository_yaml') is not None): _cli_load_invariant(kwargs.get('fn_name') is None) repo_yaml = (os.path.abspath(kwargs.get('repository_yaml')) if kwargs.get('repository_yaml') else DEFAULT_REPOSITORY_YAML_FILENAME) _cli_load_invariant( os.path.exists(repo_yaml), 'Expected to use file "{}" to load repository but it does not exist. ' 'Verify your current working directory or CLI arguments.'.format( repo_yaml), ) return ReconstructableRepository.from_legacy_repository_yaml( repo_yaml).get_reconstructable_pipeline(pipeline_name) # Pipeline from repository python file elif kwargs.get('python_file') and kwargs.get('fn_name') and pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructableRepository.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name']).get_reconstructable_pipeline(pipeline_name) # Pipeline from repository module elif kwargs.get('module_name') and kwargs.get('fn_name') and pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructableRepository.for_module( kwargs['module_name'], kwargs['fn_name']).get_reconstructable_pipeline(pipeline_name) # Pipeline from pipeline python file elif kwargs.get('python_file') and kwargs.get( 'fn_name') and not pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('module_name') is None) return ReconstructablePipeline.for_file( os.path.abspath(kwargs['python_file']), kwargs['fn_name']) # Pipeline from pipeline module elif kwargs.get('module_name') and kwargs.get( 'fn_name') and not pipeline_name: _cli_load_invariant(kwargs.get('repository_yaml') is None) _cli_load_invariant(kwargs.get('python_file') is None) return ReconstructablePipeline.for_module(kwargs['module_name'], kwargs['fn_name']) else: _cli_load_invariant(False)