def test_status_map_http_error(mock_get) -> None:
mock_response = Mock()
mock_response.ok = False
mock_get.return_value = mock_response
status_map = StatusClient()
with pytest.raises(AlgorithmError):
status_map.get_status(["algo-1"])
def test_status_map_propagate_errors(mock_get) -> None:
mock_get.side_effect = ValueError("Something went wrong")
status_map = StatusClient()
with pytest.raises(ValueError):
status_map.get_status(["algo-1", "algo-2"])
def test_status_map_one_host(mock_get) -> None:
mock_response = Mock()
mock_response.ok = True
mock_response.json.return_value = {
"status": "WAITING",
"timestamp": "2000-01-01"
}
mock_get.return_value = mock_response
status_map = StatusClient()
status_message = status_map.get_status(["algo-1"])
assert status_message["algo-1"].status == Status.WAITING
def __init__(
self,
resource_config: Dict[str, Any] = None, # type: ignore
processing_job_config: Dict[str, Any] = None, # type: ignore
) -> None:
"""Initialize a ProcessingJobManager, loading configs if not provided."""
logging.basicConfig(level=logging.INFO)
self.logger = logging.getLogger("smspark-submit")
try:
resource_config_path = "/opt/ml/config/resourceconfig.json"
with open(resource_config_path, "r") as f:
self._resource_config = json.load(f)
except Exception:
self.logger.warning(
"Could not read resource config file at {}. Using default resourceconfig.".format(resource_config_path)
)
self._resource_config = default_resource_config
self.logger.info(self._resource_config)
try:
processing_job_config_path = "/opt/ml/config/processingjobconfig.json"
with open(processing_job_config_path, "r") as f:
self._processing_job_config = json.load(f)
except Exception:
self.logger.warning(
"Could not read resource config file at {}. Using default resourceconfig.".format(resource_config_path)
)
self._processing_job_config = default_processing_job_config
self.logger.info(self._processing_job_config)
self.bootstrapper = Bootstrapper(self._resource_config)
self.waiter = Waiter()
self.status_app = StatusApp()
self.status_client = StatusClient()