def grpc_command(port=None, socket=None, host='localhost', **kwargs):
if seven.IS_WINDOWS and port is None:
raise click.UsageError(
'You must pass a valid --port/-p on Windows: --socket/-f not supported.'
)
if not (port or socket and not (port and socket)):
raise click.UsageError(
'You must pass one and only one of --port/-p or --socket/-f.')
loadable_target_origin = None
if any(kwargs[key] for key in
['attribute', 'working_directory', 'module_name', 'python_file']):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
attribute=kwargs['attribute'],
working_directory=kwargs['working_directory'],
module_name=kwargs['module_name'],
python_file=kwargs['python_file'],
)
server = DagsterGrpcServer(port=port,
socket=socket,
host=host,
loadable_target_origin=loadable_target_origin)
server.serve()
def get_bar_repo_grpc_repository_location_handle():
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
attribute='bar_repo',
python_file=file_relative_path(__file__, 'api_tests_repo.py'),
),
location_name='bar_repo',
)
def sync_get_external_pipeline_subset_ephemeral_grpc(pipeline_origin, solid_selection=None):
check.inst_param(pipeline_origin, 'pipeline_origin', PipelinePythonOrigin)
check.opt_list_param(solid_selection, 'solid_selection', of_type=str)
with ephemeral_grpc_api_client(
loadable_target_origin=LoadableTargetOrigin(executable_path=pipeline_origin.executable_path)
) as api_client:
return sync_get_external_pipeline_subset_grpc(api_client, pipeline_origin, solid_selection)
def sync_get_external_schedule_execution_data_ephemeral_grpc(
instance, repository_handle, schedule_name
):
origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
LoadableTargetOrigin(executable_path=origin.executable_path)
) as api_client:
return sync_get_external_schedule_execution_data_grpc(
api_client, instance, repository_handle, schedule_name
)
def sync_get_external_partition_names_ephemeral_grpc(repository_handle, partition_set_name):
check.inst_param(repository_handle, 'repository_handle', RepositoryHandle)
check.str_param(partition_set_name, 'partition_set_name')
repository_origin = repository_handle.get_origin()
with ephemeral_grpc_api_client(
LoadableTargetOrigin(executable_path=repository_origin.executable_path)
) as api_client:
return sync_get_external_partition_names_grpc(
api_client, repository_handle, partition_set_name
)
def get_external_pipeline_from_managed_grpc_python_env_repository(pipeline_name):
repository_location_handle = RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
attribute='nope',
python_file=file_relative_path(__file__, 'test_cli_api_run_launcher.py'),
),
location_name='nope',
)
repository_location = GrpcServerRepositoryLocation(repository_location_handle)
yield repository_location.get_repository('nope').get_full_external_pipeline(pipeline_name)
def test_heartbeat():
loadable_target_origin = LoadableTargetOrigin(
attribute='bar_repo', python_file=file_relative_path(__file__, 'grpc_repo.py'),
)
server = GrpcServerProcess(
loadable_target_origin=loadable_target_origin,
max_workers=2,
heartbeat=True,
heartbeat_timeout=1,
)
with server.create_ephemeral_client() as client:
client.heartbeat()
assert server.server_process.poll() is None
time.sleep(2)
assert server.server_process.poll() is not None
def grpc_schedule_origin(schedule_name):
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
attribute='the_repo')
with GrpcServerProcess(
loadable_target_origin=loadable_target_origin
).create_ephemeral_client() as api_client:
repo_origin = RepositoryGrpcServerOrigin(
host=api_client.host,
port=api_client.port,
socket=api_client.socket,
repository_name='the_repo',
)
yield repo_origin.get_schedule_origin(schedule_name)
def sync_list_repositories_ephemeral_grpc(executable_path, python_file,
module_name, working_directory):
from dagster.grpc.client import ephemeral_grpc_api_client
from dagster.grpc.types import LoadableTargetOrigin
check.str_param(executable_path, 'executable_path')
check.opt_str_param(python_file, 'python_file')
check.opt_str_param(module_name, 'module_name')
check.opt_str_param(working_directory, 'working_directory')
with ephemeral_grpc_api_client(loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
module_name=module_name,
python_file=python_file,
working_directory=working_directory,
attribute=None,
)) as api_client:
return sync_list_repositories_grpc(api_client)
def location_handle_from_python_file(python_file,
attribute,
user_process_api,
location_name=None,
working_directory=None):
check.str_param(python_file, 'python_file')
check.opt_str_param(attribute, 'attribute')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
check.opt_str_param(location_name, 'location_name')
check.opt_str_param(working_directory, 'working_directory')
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=sys.executable,
python_file=python_file,
module_name=None,
working_directory=working_directory,
attribute=attribute,
),
location_name=location_name,
)
loadable_targets = ([
LoadableTarget(
attribute,
load_def_in_python_file(python_file, attribute, working_directory))
] if attribute else loadable_targets_from_python_file(
python_file, working_directory))
repository_code_pointer_dict = {}
for loadable_target in loadable_targets:
repository_code_pointer_dict[loadable_target.target_definition.
name] = CodePointer.from_python_file(
python_file,
loadable_target.attribute,
working_directory)
return RepositoryLocationHandle.create_out_of_process_location(
repository_code_pointer_dict=repository_code_pointer_dict,
# default to the name of the repository symbol for now
location_name=assign_location_name(location_name,
repository_code_pointer_dict),
)
def location_handle_from_python_file(
python_file,
attribute,
user_process_api,
location_name=None,
working_directory=None,
executable_path=sys.executable,
):
check.str_param(python_file, 'python_file')
check.opt_str_param(attribute, 'attribute')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
check.opt_str_param(location_name, 'location_name')
check.opt_str_param(working_directory, 'working_directory')
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=python_file,
module_name=None,
working_directory=working_directory,
attribute=attribute,
),
location_name=location_name,
)
else:
response = sync_list_repositories(
executable_path=executable_path,
python_file=python_file,
module_name=None,
working_directory=working_directory,
attribute=attribute,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_file(python_file, lrs.attribute,
working_directory)
for lrs in response.repository_symbols
},
)
def _location_handle_from_python_environment_config(python_environment_config,
yaml_path,
user_process_api):
check.dict_param(python_environment_config, 'python_environment_config')
check.str_param(yaml_path, 'yaml_path')
check.inst_param(user_process_api, 'user_process_api', UserProcessApi)
executable_path, target_config = (
# do shell expansion on path
os.path.expanduser(python_environment_config['executable_path']),
python_environment_config['target'],
)
check.invariant(is_target_config(target_config))
python_file_config, python_module_config, python_package_config = (
target_config.get('python_file'),
target_config.get('python_module'),
target_config.get('python_package'),
)
if python_file_config:
absolute_path, attribute, location_name, working_directory = _get_python_file_config_data(
python_file_config, yaml_path)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=absolute_path,
module_name=None,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=absolute_path,
module_name=None,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_file(absolute_path, lrs.attribute,
working_directory)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute:
CodePointer.from_python_file(absolute_path, attribute,
working_directory)
},
)
elif python_module_config:
check.invariant(python_module_config)
module_name, attribute, location_name = _get_module_config_data(
python_module_config)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=module_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=module_name,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_module(module_name, lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute: CodePointer.from_module(module_name, attribute)
},
)
else:
check.invariant(python_package_config)
package_name, attribute, location_name = _get_package_config_data(
python_package_config)
if user_process_api == UserProcessApi.GRPC:
return RepositoryLocationHandle.create_process_bound_grpc_server_location(
loadable_target_origin=LoadableTargetOrigin(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
attribute=attribute,
),
location_name=location_name,
)
elif not attribute:
response = sync_list_repositories(
executable_path=executable_path,
python_file=None,
module_name=package_name,
working_directory=None,
)
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
lrs.repository_name:
CodePointer.from_python_package(package_name,
lrs.attribute)
for lrs in response.repository_symbols
},
)
else:
return RepositoryLocationHandle.create_python_env_location(
executable_path=executable_path,
location_name=location_name,
repository_code_pointer_dict={
attribute:
CodePointer.from_python_package(package_name, attribute)
},
)
def test_cancel_run():
with temp_instance() as instance:
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
python_file=__file__,
working_directory=None,
)
with GrpcServerProcess(
loadable_target_origin,
max_workers=10).create_ephemeral_client() as api_client:
streaming_results = []
pipeline_run = instance.create_run_for_pipeline(
streaming_pipeline,
run_config={
'solids': {
'streamer': {
'config': {
'length': 20
}
}
}
},
)
execute_run_args = ExecuteRunArgs(
pipeline_origin=PipelineGrpcServerOrigin(
pipeline_name='streaming_pipeline',
repository_origin=RepositoryGrpcServerOrigin(
host='localhost',
socket=api_client.socket,
port=api_client.port,
repository_name='test_repository',
),
),
pipeline_run_id=pipeline_run.run_id,
instance_ref=instance.get_ref(),
)
stream_events_result_thread = threading.Thread(
target=_stream_events_target,
args=[streaming_results, api_client, execute_run_args])
stream_events_result_thread.daemon = True
stream_events_result_thread.start()
poll_for_step_start(instance, pipeline_run.run_id)
res = api_client.cancel_execution(
cancel_execution_request=CancelExecutionRequest(
run_id=pipeline_run.run_id))
assert res.success is True
poll_for_run(instance, pipeline_run.run_id)
logs = instance.all_logs(pipeline_run.run_id)
assert (len([
ev for ev in logs
if ev.dagster_event.event_type_value == 'STEP_MATERIALIZATION'
]) < 20)
# soft termination
assert [
ev for ev in logs
if ev.dagster_event.event_type_value == 'STEP_FAILURE'
]