def host_dagit_ui_with_workspace_process_context(
workspace_process_context: WorkspaceProcessContext,
host: Optional[str],
port: int,
path_prefix: str,
):
check.inst_param(workspace_process_context, "workspace_process_context",
WorkspaceProcessContext)
check.opt_str_param(host, "host")
check.int_param(port, "port")
check.str_param(path_prefix, "path_prefix")
configure_loggers()
logger = logging.getLogger("dagit")
app = create_app_from_workspace_process_context(workspace_process_context,
path_prefix)
logger.info(
"Serving dagit on http://{host}:{port}{path_prefix} in process {pid}".
format(host=host, port=port, path_prefix=path_prefix, pid=os.getpid()))
log_action(workspace_process_context.instance, START_DAGIT_WEBSERVER)
with uploading_logging_thread():
uvicorn.run(
app,
host=host,
port=port,
access_log=False,
log_level="warning",
)
def create_test_daemon_workspace(workspace_load_target):
"""Creates a DynamicWorkspace suitable for passing into a DagsterDaemon loop when running tests."""
configure_loggers()
with create_daemon_grpc_server_registry() as grpc_server_registry:
with DaemonWorkspace(grpc_server_registry,
workspace_load_target) as workspace:
yield workspace
def start_server(instance,
host,
port,
path_prefix,
app,
port_lookup,
port_lookup_attempts=0):
server = pywsgi.WSGIServer((host, port),
app,
handler_class=WebSocketHandler)
configure_loggers()
logger = logging.getLogger("dagit")
logger.info(
"Serving dagit on http://{host}:{port}{path_prefix} in process {pid}".
format(host=host, port=port, path_prefix=path_prefix, pid=os.getpid()))
log_action(instance, START_DAGIT_WEBSERVER)
with uploading_logging_thread():
try:
server.serve_forever()
except OSError as os_error:
if "Address already in use" in str(os_error):
if port_lookup and (port_lookup_attempts > 0 or click.confirm(
("Another process on your machine is already listening on port {port}. "
"Would you like to run the app at another port instead?"
).format(port=port))):
port_lookup_attempts += 1
start_server(
instance,
host,
port + port_lookup_attempts,
path_prefix,
app,
True,
port_lookup_attempts,
)
else:
raise Exception(
f"Another process on your machine is already listening on port {port}. "
"It is possible that you have another instance of dagit "
"running somewhere using the same port. Or it could be another "
"random process. Either kill that process or use the -p option to "
"select another port.") from os_error
else:
raise os_error
def __init__(
self,
instance,
daemons,
gen_workspace,
heartbeat_interval_seconds=DEFAULT_HEARTBEAT_INTERVAL_SECONDS,
heartbeat_tolerance_seconds=DEFAULT_DAEMON_HEARTBEAT_TOLERANCE_SECONDS,
error_interval_seconds=DEFAULT_DAEMON_ERROR_INTERVAL_SECONDS,
handler="default",
):
self._daemon_uuid = str(uuid.uuid4())
self._daemons = {}
self._daemon_threads = {}
self._instance = check.inst_param(instance, "instance", DagsterInstance)
self._daemons = {
daemon.daemon_type(): daemon
for daemon in check.list_param(daemons, "daemons", of_type=DagsterDaemon)
}
self._gen_workspace = check.callable_param(gen_workspace, "gen_workspace")
self._heartbeat_interval_seconds = check.numeric_param(
heartbeat_interval_seconds, "heartbeat_interval_seconds"
)
self._heartbeat_tolerance_seconds = check.numeric_param(
heartbeat_tolerance_seconds, "heartbeat_tolerance_seconds"
)
if not self._daemons:
raise Exception("No daemons configured on the DagsterInstance")
self._daemon_shutdown_event = threading.Event()
configure_loggers(handler=handler)
self._logger = logging.getLogger("dagster.daemon")
self._logger.info(
"instance is configured with the following daemons: {}".format(
_sorted_quoted(type(daemon).__name__ for daemon in self.daemons)
)
)
self._last_healthy_heartbeat_times = {}
for daemon_type, daemon in self._daemons.items():
self._daemon_threads[daemon_type] = threading.Thread(
target=daemon.run_daemon_loop,
args=(
self._instance.get_ref(),
self._daemon_uuid,
self._daemon_shutdown_event,
gen_workspace,
heartbeat_interval_seconds,
error_interval_seconds,
),
name="dagster-daemon-{daemon_type}".format(daemon_type=daemon_type),
daemon=True, # Individual daemons should not outlive controller process
)
self._last_healthy_heartbeat_times[daemon_type] = time.time()
self._daemon_threads[daemon_type].start()
self._start_time = pendulum.now("UTC")
def grpc_command(
port=None,
socket=None,
host=None,
max_workers=None,
heartbeat=False,
heartbeat_timeout=30,
lazy_load_user_code=False,
ipc_output_file=None,
fixed_server_id=None,
override_system_timezone=None,
log_level="INFO",
use_python_environment_entry_point=False,
container_context=None,
**kwargs,
):
if seven.IS_WINDOWS and port is None:
raise click.UsageError(
"You must pass a valid --port/-p on Windows: --socket/-s not supported."
)
if not (port or socket and not (port and socket)):
raise click.UsageError(
"You must pass one and only one of --port/-p or --socket/-s.")
configure_loggers(log_level=coerce_valid_log_level(log_level))
logger = logging.getLogger("dagster.code_server")
loadable_target_origin = None
if any(kwargs[key] for key in [
"attribute",
"working_directory",
"module_name",
"package_name",
"python_file",
"empty_working_directory",
]):
loadable_target_origin = LoadableTargetOrigin(
executable_path=sys.executable,
attribute=kwargs["attribute"],
working_directory=(None if kwargs.get("empty_working_directory")
else get_working_directory_from_kwargs(kwargs)),
module_name=kwargs["module_name"],
python_file=kwargs["python_file"],
package_name=kwargs["package_name"],
)
with (mock_system_timezone(override_system_timezone)
if override_system_timezone else nullcontext()):
server = DagsterGrpcServer(
port=port,
socket=socket,
host=host,
loadable_target_origin=loadable_target_origin,
max_workers=max_workers,
heartbeat=heartbeat,
heartbeat_timeout=heartbeat_timeout,
lazy_load_user_code=lazy_load_user_code,
ipc_output_file=ipc_output_file,
fixed_server_id=fixed_server_id,
entry_point=(get_python_environment_entry_point(sys.executable)
if use_python_environment_entry_point else
DEFAULT_DAGSTER_ENTRY_POINT),
container_context=json.loads(container_context)
if container_context != None else None,
)
code_desc = " "
if loadable_target_origin:
if loadable_target_origin.python_file:
code_desc = f" for file {loadable_target_origin.python_file} "
elif loadable_target_origin.package_name:
code_desc = f" for package {loadable_target_origin.package_name} "
elif loadable_target_origin.module_name:
code_desc = f" for module {loadable_target_origin.module_name} "
server_desc = (
f"Dagster code server{code_desc}on port {port} in process {os.getpid()}"
if port else
f"Dagster code server{code_desc}in process {os.getpid()}")
logger.info("Started {server_desc}".format(server_desc=server_desc))
try:
server.serve()
finally:
logger.info(
"Shutting down {server_desc}".format(server_desc=server_desc))