def _setup_logging() -> None:
"""Log to autoscaler log file
(typically, /tmp/ray/session_latest/logs/monitor.*)
Also log to pod stdout (logs viewable with `kubectl logs <head-pod> -c autoscaler`).
"""
# Write logs at info level to monitor.log.
setup_component_logger(
logging_level=ray_constants.LOGGER_LEVEL, # info
logging_format=ray_constants.LOGGER_FORMAT,
log_dir=os.path.join(ray._private.utils.get_ray_temp_dir(),
ray.node.SESSION_LATEST, "logs"),
filename=ray_constants.MONITOR_LOG_FILE_NAME, # monitor.log
max_bytes=ray_constants.LOGGING_ROTATE_BYTES,
backup_count=ray_constants.LOGGING_ROTATE_BACKUP_COUNT,
)
# Also log to stdout for debugging with `kubectl logs`.
root_logger = logging.getLogger("")
root_logger.setLevel(logging.INFO)
root_handler = logging.StreamHandler()
root_handler.setLevel(logging.INFO)
root_handler.setFormatter(logging.Formatter(ray_constants.LOGGER_FORMAT))
root_logger.addHandler(root_handler)
def _setup_logging() -> None:
"""Log to autoscaler log file
(typically, /tmp/ray/session_latest/logs/monitor.*)
Also log to pod stdout (logs viewable with `kubectl logs <head-pod> -c autoscaler`).
"""
# Write logs at info level to monitor.log.
setup_component_logger(
logging_level=ray_constants.LOGGER_LEVEL, # info
logging_format=ray_constants.LOGGER_FORMAT,
log_dir=os.path.join(ray._private.utils.get_ray_temp_dir(),
ray.node.SESSION_LATEST, "logs"),
filename=ray_constants.MONITOR_LOG_FILE_NAME, # monitor.log
max_bytes=ray_constants.LOGGING_ROTATE_BYTES,
backup_count=ray_constants.LOGGING_ROTATE_BACKUP_COUNT,
logger_name="ray", # Root of the logging hierachy for Ray code.
)
def get_or_create_logger(self, job_id: bytes):
job_id = job_id.decode()
if job_id not in self._per_job_logger_cache:
params = self._logging_params.copy()
params["filename"] = f"runtime_env_setup-{job_id}.log"
params["logger_name"] = f"runtime_env_{job_id}"
per_job_logger = setup_component_logger(**params)
self._per_job_logger_cache[job_id] = per_job_logger
return self._per_job_logger_cache[job_id]
required=False,
type=int,
default=ray_constants.LOGGING_ROTATE_BACKUP_COUNT,
help="Specify the backup count of rotated log file, default is "
f"{ray_constants.LOGGING_ROTATE_BACKUP_COUNT}.")
parser.add_argument(
"--monitor-ip",
required=False,
type=str,
default=None,
help="The IP address of the machine hosting the monitor process.")
args = parser.parse_args()
setup_component_logger(
logging_level=args.logging_level,
logging_format=args.logging_format,
log_dir=args.logs_dir,
filename=args.logging_filename,
max_bytes=args.logging_rotate_bytes,
backup_count=args.logging_rotate_backup_count)
logger.info(f"Starting monitor using ray installation: {ray.__file__}")
logger.info(f"Ray version: {ray.__version__}")
logger.info(f"Ray commit: {ray.__commit__}")
logger.info(f"Monitor started with command: {sys.argv}")
if args.autoscaling_config:
autoscaling_config = os.path.expanduser(args.autoscaling_config)
else:
autoscaling_config = None
monitor = Monitor(
parser.add_argument(
"--runtime-env-dir",
required=True,
type=str,
default=None,
help="Specify the path of the resource directory used by runtime_env.")
args = parser.parse_args()
try:
logging_params = dict(logging_level=args.logging_level,
logging_format=args.logging_format,
log_dir=args.log_dir,
filename=args.logging_filename,
max_bytes=args.logging_rotate_bytes,
backup_count=args.logging_rotate_backup_count)
setup_component_logger(**logging_params)
agent = DashboardAgent(args.node_ip_address,
args.redis_address,
args.dashboard_agent_port,
args.gcs_address,
redis_password=args.redis_password,
temp_dir=args.temp_dir,
session_dir=args.session_dir,
runtime_env_dir=args.runtime_env_dir,
log_dir=args.log_dir,
metrics_export_port=args.metrics_export_port,
node_manager_port=args.node_manager_port,
listen_port=args.listen_port,
object_store_name=args.object_store_name,
raylet_name=args.raylet_name,