def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
fixed_server_id=None,
startup_timeout=20,
):
check.invariant((port or socket) and not (port and socket), "Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.opt_int_param(max_workers, "max_workers")
from dagster.core.test_utils import get_mocked_system_timezone
mocked_system_timezone = get_mocked_system_timezone()
executable_path = loadable_target_origin.executable_path if loadable_target_origin else None
subprocess_args = (
(
get_python_environment_entry_point(executable_path)
if executable_path
else DEFAULT_DAGSTER_ENTRY_POINT
)
+ ["api", "grpc"]
+ ["--lazy-load-user-code"]
+ (["--port", str(port)] if port else [])
+ (["--socket", socket] if socket else [])
+ (["-n", str(max_workers)] if max_workers else [])
+ (["--heartbeat"] if heartbeat else [])
+ (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else [])
+ (["--fixed-server-id", fixed_server_id] if fixed_server_id else [])
+ (["--override-system-timezone", mocked_system_timezone] if mocked_system_timezone else [])
+ (["--log-level", "WARNING"]) # don't log INFO messages for automatically spun up servers
+ (["--use-python-environment-entry-point"] if executable_path else [])
)
if loadable_target_origin:
subprocess_args += loadable_target_origin.get_cli_args()
server_process = open_ipc_subprocess(subprocess_args)
from dagster.grpc.client import DagsterGrpcClient
client = DagsterGrpcClient(
port=port,
socket=socket,
host="localhost",
)
try:
wait_for_grpc_server(server_process, client, subprocess_args, timeout=startup_timeout)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=1,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
fixed_server_id=None,
):
check.invariant((port or socket) and not (port and socket), "Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin", LoadableTargetOrigin)
check.int_param(max_workers, "max_workers")
from dagster.core.test_utils import get_mocked_system_timezone
with seven.TemporaryDirectory() as temp_dir:
output_file = os.path.join(
temp_dir, "grpc-server-startup-{uuid}".format(uuid=uuid.uuid4().hex)
)
mocked_system_timezone = get_mocked_system_timezone()
subprocess_args = (
[
loadable_target_origin.executable_path
if loadable_target_origin and loadable_target_origin.executable_path
else sys.executable,
"-m",
"dagster.grpc",
]
+ (["--port", str(port)] if port else [])
+ (["--socket", socket] if socket else [])
+ ["-n", str(max_workers)]
+ (["--heartbeat"] if heartbeat else [])
+ (["--heartbeat-timeout", str(heartbeat_timeout)] if heartbeat_timeout else [])
+ (["--lazy-load-user-code"] if lazy_load_user_code else [])
+ (["--ipc-output-file", output_file])
+ (["--fixed-server-id", fixed_server_id] if fixed_server_id else [])
+ (
["--override-system-timezone", mocked_system_timezone]
if mocked_system_timezone
else []
)
)
if loadable_target_origin:
subprocess_args += loadable_target_origin.get_cli_args()
server_process = open_ipc_subprocess(subprocess_args)
try:
wait_for_grpc_server(server_process, output_file)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def open_server_process(
port,
socket,
loadable_target_origin=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
fixed_server_id=None,
):
check.invariant((port or socket) and not (port and socket),
"Set only port or socket")
check.opt_inst_param(loadable_target_origin, "loadable_target_origin",
LoadableTargetOrigin)
check.opt_int_param(max_workers, "max_workers")
from dagster.core.test_utils import get_mocked_system_timezone
mocked_system_timezone = get_mocked_system_timezone()
subprocess_args = (
[
loadable_target_origin.executable_path if loadable_target_origin
and loadable_target_origin.executable_path else sys.executable,
"-m",
"dagster.grpc",
] + ["--lazy-load-user-code"] +
(["--port", str(port)] if port else []) +
(["--socket", socket] if socket else []) +
(["-n", str(max_workers)] if max_workers else []) +
(["--heartbeat"] if heartbeat else []) +
(["--heartbeat-timeout", str(heartbeat_timeout)]
if heartbeat_timeout else []) +
(["--fixed-server-id", fixed_server_id] if fixed_server_id else []) +
(["--override-system-timezone", mocked_system_timezone]
if mocked_system_timezone else []))
if loadable_target_origin:
subprocess_args += loadable_target_origin.get_cli_args()
server_process = open_ipc_subprocess(subprocess_args)
from dagster.grpc.client import DagsterGrpcClient
client = DagsterGrpcClient(
port=port,
socket=socket,
host="localhost",
)
try:
wait_for_grpc_server(server_process, client, subprocess_args)
except:
if server_process.poll() is None:
server_process.terminate()
raise
return server_process
def _write_bash_script_to_file(self, instance, external_schedule):
# Get path to store bash script
schedule_origin_id = external_schedule.get_external_origin_id()
script_file = self._get_bash_script_file_path(instance, schedule_origin_id)
# Get path to store schedule attempt logs
logs_directory = self._get_or_create_logs_directory(instance, schedule_origin_id)
schedule_log_file_name = "{}_{}.result".format("${RUN_DATE}", schedule_origin_id)
schedule_log_file_path = os.path.join(logs_directory, schedule_log_file_name)
local_target = external_schedule.get_external_origin()
# Environment information needed for execution
dagster_home = os.getenv("DAGSTER_HOME")
override_system_timezone = get_mocked_system_timezone()
script_contents = """
#!/bin/bash
export DAGSTER_HOME={dagster_home}
export LANG=en_US.UTF-8
{env_vars}
export RUN_DATE=$(date "+%Y%m%dT%H%M%S")
{python_exe} -m dagster api launch_scheduled_execution --schedule_name {schedule_name} {repo_cli_args} {override_timezone_args} "{result_file}"
""".format(
python_exe=sys.executable,
schedule_name=external_schedule.name,
repo_cli_args=local_target.get_repo_cli_args(),
override_timezone_args=(
f"--override-system-timezone={override_system_timezone}"
if override_system_timezone
else ""
),
result_file=schedule_log_file_path,
dagster_home=dagster_home,
env_vars="\n".join(
[
"export {key}={value}".format(key=key, value=value)
for key, value in external_schedule.environment_vars.items()
]
),
)
with io.open(script_file, "w", encoding="utf-8") as f:
f.write(script_contents)
st = os.stat(script_file)
os.chmod(script_file, st.st_mode | stat.S_IEXEC)
return script_file
