def open_server_process( port, socket, loadable_target_origin=None, max_workers=None, heartbeat=False, heartbeat_timeout=30, fixed_server_id=None, startup_timeout=20, ): check.invariant((port or socket) and not (port and socket), "Set only port or socket") check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin) check.opt_int_param(max_workers, "max_workers") from dagster.core.test_utils import get_mocked_system_timezone mocked_system_timezone = get_mocked_system_timezone() executable_path = loadable_target_origin.executable_path if loadable_target_origin else None subprocess_args = ( ( get_python_environment_entry_point(executable_path) if executable_path else DEFAULT_DAGSTER_ENTRY_POINT ) + ["api", "grpc"] + ["--lazy-load-user-code"] + (["--port", str(port)] if port else []) + (["--socket", socket] if socket else []) + (["-n", str(max_workers)] if max_workers else []) + (["--heartbeat"] if heartbeat else []) + (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else []) + (["--fixed-server-id", fixed_server_id] if fixed_server_id else []) + (["--override-system-timezone", mocked_system_timezone] if mocked_system_timezone else []) + (["--log-level", "WARNING"]) # don't log INFO messages for automatically spun up servers + (["--use-python-environment-entry-point"] if executable_path else []) ) if loadable_target_origin: subprocess_args += loadable_target_origin.get_cli_args() server_process = open_ipc_subprocess(subprocess_args) from dagster.grpc.client import DagsterGrpcClient client = DagsterGrpcClient( port=port, socket=socket, host="localhost", ) try: wait_for_grpc_server(server_process, client, subprocess_args, timeout=startup_timeout) except: if server_process.poll() is None: server_process.terminate() raise return server_process
def open_server_process( port, socket, loadable_target_origin=None, max_workers=1, heartbeat=False, heartbeat_timeout=30, lazy_load_user_code=False, fixed_server_id=None, ): check.invariant((port or socket) and not (port and socket), "Set only port or socket") check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin) check.int_param(max_workers, "max_workers") from dagster.core.test_utils import get_mocked_system_timezone with seven.TemporaryDirectory() as temp_dir: output_file = os.path.join( temp_dir, "grpc-server-startup-{uuid}".format(uuid=uuid.uuid4().hex) ) mocked_system_timezone = get_mocked_system_timezone() subprocess_args = ( [ loadable_target_origin.executable_path if loadable_target_origin and loadable_target_origin.executable_path else sys.executable, "-m", "dagster.grpc", ] + (["--port", str(port)] if port else []) + (["--socket", socket] if socket else []) + ["-n", str(max_workers)] + (["--heartbeat"] if heartbeat else []) + (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else []) + (["--lazy-load-user-code"] if lazy_load_user_code else []) + (["--ipc-output-file", output_file]) + (["--fixed-server-id", fixed_server_id] if fixed_server_id else []) + ( ["--override-system-timezone", mocked_system_timezone] if mocked_system_timezone else [] ) ) if loadable_target_origin: subprocess_args += loadable_target_origin.get_cli_args() server_process = open_ipc_subprocess(subprocess_args) try: wait_for_grpc_server(server_process, output_file) except: if server_process.poll() is None: server_process.terminate() raise return server_process
def open_server_process( port, socket, loadable_target_origin=None, max_workers=None, heartbeat=False, heartbeat_timeout=30, fixed_server_id=None, ): check.invariant((port or socket) and not (port and socket), "Set only port or socket") check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin) check.opt_int_param(max_workers, "max_workers") from dagster.core.test_utils import get_mocked_system_timezone mocked_system_timezone = get_mocked_system_timezone() subprocess_args = ( [ loadable_target_origin.executable_path if loadable_target_origin and loadable_target_origin.executable_path else sys.executable, "-m", "dagster.grpc", ] + ["--lazy-load-user-code"] + (["--port", str(port)] if port else []) + (["--socket", socket] if socket else []) + (["-n", str(max_workers)] if max_workers else []) + (["--heartbeat"] if heartbeat else []) + (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else []) + (["--fixed-server-id", fixed_server_id] if fixed_server_id else []) + (["--override-system-timezone", mocked_system_timezone] if mocked_system_timezone else [])) if loadable_target_origin: subprocess_args += loadable_target_origin.get_cli_args() server_process = open_ipc_subprocess(subprocess_args) from dagster.grpc.client import DagsterGrpcClient client = DagsterGrpcClient( port=port, socket=socket, host="localhost", ) try: wait_for_grpc_server(server_process, client, subprocess_args) except: if server_process.poll() is None: server_process.terminate() raise return server_process
def _write_bash_script_to_file(self, instance, external_schedule): # Get path to store bash script schedule_origin_id = external_schedule.get_external_origin_id() script_file = self._get_bash_script_file_path(instance, schedule_origin_id) # Get path to store schedule attempt logs logs_directory = self._get_or_create_logs_directory(instance, schedule_origin_id) schedule_log_file_name = "{}_{}.result".format("${RUN_DATE}", schedule_origin_id) schedule_log_file_path = os.path.join(logs_directory, schedule_log_file_name) local_target = external_schedule.get_external_origin() # Environment information needed for execution dagster_home = os.getenv("DAGSTER_HOME") override_system_timezone = get_mocked_system_timezone() script_contents = """ #!/bin/bash export DAGSTER_HOME={dagster_home} export LANG=en_US.UTF-8 {env_vars} export RUN_DATE=$(date "+%Y%m%dT%H%M%S") {python_exe} -m dagster api launch_scheduled_execution --schedule_name {schedule_name} {repo_cli_args} {override_timezone_args} "{result_file}" """.format( python_exe=sys.executable, schedule_name=external_schedule.name, repo_cli_args=local_target.get_repo_cli_args(), override_timezone_args=( f"--override-system-timezone={override_system_timezone}" if override_system_timezone else "" ), result_file=schedule_log_file_path, dagster_home=dagster_home, env_vars="\n".join( [ "export {key}={value}".format(key=key, value=value) for key, value in external_schedule.environment_vars.items() ] ), ) with io.open(script_file, "w", encoding="utf-8") as f: f.write(script_contents) st = os.stat(script_file) os.chmod(script_file, st.st_mode | stat.S_IEXEC) return script_file