def _mgr_fn(recon_repo): """Goes out of process but same process as host process""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) with Workspace( [ ManagedGrpcPythonEnvRepositoryLocationOrigin( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path(__file__, "setup.py"), attribute="test_repo", ), location_name="test", ), ManagedGrpcPythonEnvRepositoryLocationOrigin( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path(__file__, "setup.py"), attribute="empty_repo", ), location_name="empty_repo", ), ] ) as workspace: yield workspace
def dagit_debug_command(input_files, port): debug_payloads = [] for input_file in input_files: click.echo("Loading {} ...".format(input_file)) with GzipFile(input_file, "rb") as file: blob = file.read().decode() debug_payload = deserialize_json_to_dagster_namedtuple(blob) check.invariant(isinstance(debug_payload, DebugRunPayload)) click.echo( "\trun_id: {} \n\tdagster version: {}".format( debug_payload.pipeline_run.run_id, debug_payload.version ) ) debug_payloads.append(debug_payload) instance = DagsterInstance.ephemeral(preload=debug_payloads) host_dagit_ui_with_workspace( workspace=Workspace([]), instance=instance, port=port, port_lookup=True, host=DEFAULT_DAGIT_HOST, path_prefix="", )
def _mgr_fn(recon_repo): """Goes out of process but same process as host process""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) empty_repo = ReconstructableRepository.from_legacy_repository_yaml( file_relative_path(__file__, "empty_repo.yaml")) with Workspace([ RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path( __file__, "setup.py"), attribute="test_repo", ), location_name="test", ), RepositoryLocationHandle.create_in_process_location( empty_repo.pointer), RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=file_relative_path( __file__, "setup.py"), attribute="empty_repo", ), location_name="empty_repo", ), ]) as workspace: yield workspace
def _mgr_fn(recon_repo): check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) with Workspace([ RepositoryLocationHandle.create_in_process_location( recon_repo.pointer) ]) as workspace: yield workspace
def define_out_of_process_workspace(python_file, fn_name): return Workspace( PythonFileTarget( python_file=python_file, attribute=fn_name, working_directory=None, location_name=main_repo_location_name(), ))
def define_test_snapshot_context(): return DagsterGraphQLContext( instance=DagsterInstance.ephemeral(), workspace=Workspace([ RepositoryLocationHandle.create_in_process_location( create_main_recon_repo().pointer) ]), )
def define_context_for_file(python_file, fn_name, instance): check.inst_param(instance, "instance", DagsterInstance) return DagsterGraphQLContext( workspace=Workspace([ RepositoryLocationHandle.create_in_process_location( CodePointer.from_python_file(python_file, fn_name, None)) ]), instance=instance, )
def define_context_for_repository_yaml(path, instance): check.inst_param(instance, "instance", DagsterInstance) return DagsterGraphQLContext( workspace=Workspace([ RepositoryLocationHandle.create_in_process_location( ReconstructableRepository.from_legacy_repository_yaml( path).pointer) ]), instance=instance, )
def _mgr_fn(recon_repo): """Goes out of process but same process as host process""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) with Workspace( WorkspaceFileTarget(paths=[ file_relative_path(__file__, "multi_location.yaml") ])) as workspace: yield workspace
def define_in_process_context(python_file, fn_name, instance): check.inst_param(instance, "instance", DagsterInstance) return WorkspaceProcessContext( workspace=Workspace([ InProcessRepositoryLocationOrigin( ReconstructableRepository.for_file(python_file, fn_name)) ]), instance=instance, ).create_request_context()
def define_out_of_process_workspace(python_file, fn_name): return Workspace([ ManagedGrpcPythonEnvRepositoryLocationOrigin( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=python_file, attribute=fn_name, ), location_name=main_repo_location_name(), ) ])
def _mgr_fn(recon_repo): """Goes out of process but same process as host process""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) with Workspace( PythonFileTarget( python_file=file_relative_path(__file__, "setup.py"), attribute="test_dict_repo", working_directory=None, location_name="test", )) as workspace: yield workspace
def _mgr_fn(recon_repo): """Goes out of process via grpc""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) loadable_target_origin = recon_repo.get_origin().loadable_target_origin with Workspace( [ ManagedGrpcPythonEnvRepositoryLocationOrigin( loadable_target_origin=loadable_target_origin, location_name="test", ) ] ) as workspace: yield workspace
def _mgr_fn(recon_repo): """Goes out of process via grpc""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) loadable_target_origin = LoadableTargetOrigin.from_python_origin( recon_repo.get_origin()) with Workspace([ RepositoryLocationHandle. create_process_bound_grpc_server_location( loadable_target_origin=loadable_target_origin, location_name="test", ) ]) as workspace: yield workspace
def _mgr_fn(recon_repo): """Goes out of process but same process as host process""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) # this is "ok" because we know the test host process containers the user code loadable_target_origin = LoadableTargetOrigin.from_python_origin( recon_repo.get_origin()) with Workspace([ RepositoryLocationHandle.create_python_env_location( loadable_target_origin=loadable_target_origin, location_name="test", ) ]) as workspace: yield workspace
def define_out_of_process_context(python_file, fn_name, instance): check.inst_param(instance, "instance", DagsterInstance) return DagsterGraphQLContext( workspace=Workspace([ RepositoryLocationHandle.create_python_env_location( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=python_file, attribute=fn_name, ), location_name="test_location", ) ]), instance=instance, )
def define_out_of_process_context(python_file, fn_name, instance): check.inst_param(instance, "instance", DagsterInstance) with Workspace([ ManagedGrpcPythonEnvRepositoryLocationOrigin( loadable_target_origin=LoadableTargetOrigin( executable_path=sys.executable, python_file=python_file, attribute=fn_name, ), location_name=main_repo_location_name(), ) ]) as workspace: yield DagsterGraphQLContext( workspace=workspace, instance=instance, )
def _mgr_fn(recon_repo): """Goes out of process via grpc""" check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) loadable_target_origin = recon_repo.get_python_origin( ).loadable_target_origin with Workspace((PythonFileTarget( python_file=loadable_target_origin.python_file, attribute=loadable_target_origin.attribute, working_directory=loadable_target_origin.working_directory, location_name="test", ) if loadable_target_origin.python_file else ModuleTarget( module_name=loadable_target_origin.module_name, attribute=loadable_target_origin.attribute, location_name="test", ))) as workspace: yield workspace
def _mgr_fn(recon_repo): check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) loadable_target_origin = recon_repo.get_python_origin( ).loadable_target_origin server_process = GrpcServerProcess( loadable_target_origin=loadable_target_origin) try: with server_process.create_ephemeral_client() as api_client: with Workspace( GrpcServerTarget( port=api_client.port, socket=api_client.socket, host=api_client.host, location_name="test", )) as workspace: yield workspace finally: server_process.wait()
def test_execute_hammer_through_dagit(): recon_repo = ReconstructableRepository.for_file( file_relative_path( __file__, "../../../dagster-test/dagster_test/toys/hammer.py"), "hammer_pipeline", ) instance = DagsterInstance.local_temp() context = DagsterGraphQLContext( workspace=Workspace([ RepositoryLocationHandle.create_in_process_location( recon_repo.pointer) ]), instance=instance, ) selector = infer_pipeline_selector(context, "hammer_pipeline") executor = SyncExecutor() variables = { "executionParams": { "runConfigData": { "storage": { "filesystem": {} }, "execution": { "dask": { "config": { "cluster": { "local": {} } } } }, }, "selector": selector, "mode": "default", } } start_pipeline_result = graphql( request_string=LAUNCH_PIPELINE_EXECUTION_MUTATION, schema=create_schema(), context=context, variables=variables, executor=executor, ) if start_pipeline_result.errors: raise Exception("{}".format(start_pipeline_result.errors)) run_id = start_pipeline_result.data["launchPipelineExecution"]["run"][ "runId"] context.drain_outstanding_executions() subscription = execute_dagster_graphql(context, SUBSCRIPTION_QUERY, variables={"runId": run_id}) subscribe_results = [] subscription.subscribe(subscribe_results.append) messages = [ x["__typename"] for x in subscribe_results[0].data["pipelineRunLogs"]["messages"] ] assert "PipelineStartEvent" in messages assert "PipelineSuccessEvent" in messages
def _mgr_fn(recon_repo): check.inst_param(recon_repo, "recon_repo", ReconstructableRepository) with Workspace([InProcessRepositoryLocationOrigin(recon_repo)]) as workspace: yield workspace