def test_handle_request_2_e(self):
"""
Test that the ``handle_request()`` method returns a success response has a valid serialized
:class:`SchedulerRequestResponse` embedded in the data attribute, and that this scheduler response is also set
as successful, when the session and authorization are good, and the request to the scheduler is appropriate and
successful (simulated in testing via a dummy scheduler client).
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
dummy_scheduler_client = DummySchedulerClient(test_successful=True)
self.handler._scheduler_client = dummy_scheduler_client
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
data_key = NWMRequestResponse.get_data_dict_key_for_scheduler_response(
)
deserialized_json_dict = response.data[data_key]
scheduler_response = SchedulerRequestResponse.factory_init_from_deserialized_json(
deserialized_json_dict)
self.assertTrue(scheduler_response.success)
def setUp(self) -> None:
self._nwm_model_request = NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"domain": "blah",
"parameters": {}
}
},
"session-secret":
"f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
})
self._example_jobs = []
self._example_jobs.append(
JobImpl(4,
1000,
model_request=self._nwm_model_request,
allocation_paradigm='single-node'))
self._uuid_str_vals = []
self._uuid_str_vals.append('12345678-1234-5678-1234-567812345678')
self._resource_allocations = []
self._resource_allocations.append(
ResourceAllocation('node001', 'node001', 4, 1000))
def mock_job(cpus: int = 4,
mem: int = 500000,
strategy: str = "single_node",
allocations: int = 0) -> RequestedJob:
"""
Generate a mock job with given cpu request
"""
# Example 0
request_string = _request_string.format(cpus=cpus, mem=mem)
request_json = _request_json
request_json['cpus'] = cpus
request_json['mem'] = mem
model_request = NWMRequest.factory_init_from_deserialized_json(
request_json)
schedule_request = SchedulerRequestMessage(model_request=model_request,
user_id=request_json['user_id'],
cpus=cpus,
mem=mem,
allocation_paradigm=strategy)
mock_job = RequestedJob(schedule_request)
allocs = []
for i in range(1, allocations + 1):
allocs.append(ResourceAllocation(i, 'hostname{}'.format(i), cpus, mem))
mock_job.allocations = allocs
return mock_job
def setUp(self) -> None:
self._example_jobs = []
model_request_0 = NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"domain": "blah",
"parameters": {}
}
},
"session-secret":
"f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
})
self._example_jobs.append(
JobImpl(cpu_count=4,
memory_size=1000,
model_request=model_request_0,
allocation_paradigm='single-node'))
scheduler_request = SchedulerRequestMessage(
model_request=NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"domain": "blah",
"parameters": {}
}
},
"session-secret":
"123f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
}),
user_id='someone',
cpus=4,
mem=500000,
allocation_paradigm='single-node')
self._example_jobs.append(RequestedJob(job_request=scheduler_request))
def test_handle_request_1_a(self):
"""
Test that the ``handle_request()`` method returns a failure response if the session for the secret cannot be
found.
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
# Now, remove the session from the manager
self.session_manager.remove_session(session)
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
self.assertFalse(response.success)
def test_handle_request_3_c(self):
"""
Test that the ``handle_request()`` method returns a failure response with the error job_id value when the
session and authorization are good, but the request to the scheduler is not successful (simulated in testing via
a dummy scheduler client).
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
dummy_scheduler_client = DummySchedulerClient(test_successful=False)
self.handler._scheduler_client = dummy_scheduler_client
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
self.assertEquals(response.job_id, -1)
def test_handle_request_2_a(self):
"""
Test that the ``handle_request()`` method returns a success response when the session and authorization are
good, and the request to the scheduler is appropriate and successful (simulated in testing via a dummy scheduler
client).
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
dummy_scheduler_client = DummySchedulerClient(test_successful=True)
self.handler._scheduler_client = dummy_scheduler_client
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
self.assertTrue(response.success)
def test_handle_request_1_c(self):
"""
Test that the ``handle_request()`` method returns an appropriate failure response with the correct failure
reason if the user is not authorized.
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
#self.session_manager._authorizer = self.fail_authorizer
self.handler._authorizer = self.fail_authorizer
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
self.assertEquals(response.reason,
InitRequestResponseReason.UNAUTHORIZED.name)
def test_handle_request_1_b(self):
"""
Test that the ``handle_request()`` method returns an appropriate failure response with the correct failure
reason if the session for the secret cannot be found.
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
# Now, remove the session from the manager
self.session_manager.remove_session(session)
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
self.assertEqual(
response.reason,
InitRequestResponseReason.UNRECOGNIZED_SESSION_SECRET.name)
def test_handle_request_3_d(self):
"""
Test that the ``handle_request()`` method returns a failure response with a valid serialized
:class:`SchedulerRequestResponse` embedded in the data attribute, when the session and authorization are good,
but the request to the scheduler are not successful (simulated in testing via a dummy scheduler client).
"""
ip_addr = self._session_ip_1
user = self._user_1
session = self.session_manager.create_session(ip_address=ip_addr,
username=user)
request = NWMRequest(session_secret=session.session_secret,
version=2.0)
dummy_scheduler_client = DummySchedulerClient(test_successful=False)
self.handler._scheduler_client = dummy_scheduler_client
response = asyncio.run(self.handler.handle_request(request=request),
debug=True)
sched_resp = SchedulerRequestResponse.factory_init_from_deserialized_json(
response.data['scheduler_response'])
self.assertTrue(isinstance(sched_resp, SchedulerRequestResponse))
def mock_job(model: str = 'nwm',
cpus: int = 4,
mem: int = 500000,
strategy: str = "single_node",
allocations: int = 0) -> RequestedJob:
"""
Generate a mock job with given cpu request
"""
request_json = _request_json
request_json['cpus'] = cpus
request_json['mem'] = mem
if model == 'nwm':
request_json['model'] = _nwm_model
model_request = NWMRequest.factory_init_from_deserialized_json(
request_json)
elif model == 'ngen':
request_json['model'] = _ngen_model
model_request = NGENRequest.factory_init_from_deserialized_json(
request_json)
else:
raise (ValueError("Unsupported mock model {}".format(model)))
schedule_request = SchedulerRequestMessage(model_request=model_request,
user_id=request_json['user_id'],
cpus=cpus,
mem=mem,
allocation_paradigm=strategy)
mock_job = RequestedJob(schedule_request)
#mock_job.job_id = uuid4()
allocs = []
for i in range(1, allocations + 1):
allocs.append(ResourceAllocation(i, 'hostname{}'.format(i), cpus, mem))
mock_job.allocations = allocs
return mock_job
def _deserialize_model_request(self, model_request_hash: dict,
parameters_hash: dict,
**kwargs) -> MaaSRequest:
"""
Deserialized the serialized data for a model request, in the form of Redis hashes/mappings, into a
::class:`MaaSRequest` object.
Essentially, this performs the reverse of ::method:`serialize_model_request`.
The method supports optional keyword arguments. In particular, these are intended to provide future support for
controlling the specific subtype of ::class:`MaaSRequest` that is created. For now, only ::class:`NWMRequest`
objects are supported.
Parameters
----------
model_request_hash : dict
The Redis hash value for the inner ::class:`MaaSRequest` object of the scheduler request.
parameters_hash : dict
The ``parameters`` of the inner ::class:`MaaSRequest` object of the request.
kwargs
Optional keyword arguments.
Returns
-------
MaaSRequest
The deserialize model request object.
"""
# TODO: consider whether there needs to be any conversion done to values (e.g., integer string to actual int)
parameters = parameters_hash
return NWMRequest(session_secret=model_request_hash['session_secret'],
version=float(model_request_hash['version']),
output=model_request_hash['output'],
parameters=parameters)
def setUp(self) -> None:
self._resource_manager = MockResourceManager()
self._resource_manager.set_resources(mock_resources())
# TODO: make sure this doesn't cause a problem
self._launcher = None
self._sample_job_requests = []
self._sample_job_requests.append(
SchedulerRequestMessage(
model_request=NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"parameters": {},
"domain": "test-domain"
}
},
"session-secret":
"f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
}),
user_id='someone',
cpus=4,
mem=500000,
allocation_paradigm='single-node'))
self._sample_job_requests.append(
SchedulerRequestMessage(
model_request=NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"parameters": {},
"domain": "test-domain"
}
},
"session-secret":
"123f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
}),
user_id='someone',
cpus=4,
mem=500000,
allocation_paradigm='single-node'))
# indexes 2 and 3 are the same as the job at index 0, except with the two other allocation paradigms
self._sample_job_requests.append(
SchedulerRequestMessage(
model_request=NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"parameters": {},
"domain": "test-domain"
}
},
"session-secret":
"f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
}),
user_id='someone',
cpus=4,
mem=500000,
allocation_paradigm='fill-nodes'))
self._sample_job_requests.append(
SchedulerRequestMessage(
model_request=NWMRequest.factory_init_from_deserialized_json({
"model": {
"nwm": {
"version": 2.0,
"output": "streamflow",
"parameters": {},
"domain": "test-domain"
}
},
"session-secret":
"f21f27ac3d443c0948aab924bddefc64891c455a756ca77a4d86ec2f697cd13c"
}),
user_id='someone',
cpus=4,
mem=500000,
allocation_paradigm='round-robin'))
self._uuid_str_vals = []
#self._uuid_str_vals.append('12345678-1234-5678-1234-567812345678')
self._uuid_str_vals.append('00000000-0000-0000-0000-000000000000')
self._uuid_str_vals.append('00000000-0000-0000-0000-000000000001')
self._uuid_str_vals.append('00000000-0000-0000-0000-000000000002')
self._uuid_str_vals.append('00000000-0000-0000-0000-000000000003')
# TODO: set these correctly
self.redis_test_host = '127.0.0.1'
self._env_type = 'test'
self._job_manager = RedisBackedJobManager(
resource_manager=self._resource_manager,
launcher=self._launcher,
redis_host=self.redis_test_host,
redis_port=self.redis_test_port,
redis_pass=self.redis_test_pass,
type=self._env_type)
async def handle_request(self, request: NWMRequest,
**kwargs) -> NWMRequestResponse:
"""
Handle the given request for a new NWM job execution and return the resulting response.
Parameters
----------
request: NWMRequest
A ``NWMRequest`` message instance with details of the job being requested.
Returns
-------
response: NWMRequestResponse
An appropriate ``NWMRequestResponse`` object.
"""
session = self._session_manager.lookup_session_by_secret(
request.session_secret)
if session is None:
reason = InitRequestResponseReason.UNRECOGNIZED_SESSION_SECRET
msg = 'Request {} does not correspond to a known authenticated session'.format(
request.to_json())
elif not await self._is_authorized(request=request, session=session):
reason = InitRequestResponseReason.UNAUTHORIZED
msg = 'User {} in session [{}] not authorized for NWM job request {}'.format(
session.user, str(session.session_id), request.to_json())
logging.debug("*************" + msg)
else:
# The context manager manages a SINGLE connection to the scheduler server
# Adhoc calls to the scheduler can be made for this connection via the scheduler_client
# These adhoc calls will use the SAME connection the context was initialized with
logging.debug("************* Preparing scheduler request message")
scheduler_message = SchedulerRequestMessage(model_request=request,
user_id=session.user)
logging.debug(
"************* Scheduler request message ready:\n{}".format(
str(scheduler_message)))
# Should be able to do this to reuse same object/context/connection across tasks, even from other methods
async with self._scheduler_client as scheduler_client:
initial_response = await scheduler_client.async_make_request(
scheduler_message)
logging.debug(
"************* Scheduler client received response:\n{}".
format(str(initial_response)))
if initial_response.success:
job_id = initial_response.job_id
#async for response in scheduler_client.get_results():
# logging.debug("************* Results:\n{}".format(str(response)))
# print(response)
# TODO: consider registering the job and relationship with session, etc.
success = initial_response.success
success_str = 'Success' if success else 'Failure'
reason = InitRequestResponseReason.ACCEPTED if success else InitRequestResponseReason.REJECTED
mesg = '{} submitting job to scheduler (returned id {})'.format(
success_str, str(initial_response.job_id))
# TODO: right now, the only supported MaaSRequest we will see is a NWMRequest, but account for other things
return NWMRequestResponse(success=success,
reason=reason.name,
message=mesg,
scheduler_response=initial_response)
# If we didn't just return by executing 'else' condition above (i.e., we don't have an authorized session) ...
return NWMRequestResponse(success=False,
reason=reason.name,
message=msg)