def __do_request(self, method, uri):
"""
Creates an HTTP request and invokes the image streaming service
:param method Method to be used by the HTTP request (GET, POST, DELETE, etc)
:param uri JSON formatted URI to attach to the HTTP request
:return 200 if successful, Error code and message otherwise
"""
try:
url = settings.IMAGE_STREAMING_SERVICE_URL + '/route'
log.info(1, '__do_request(' + method + ', ' + url + ')')
headers = {
'Content-Type': 'application/json',
'Cookie': COOKIE_ID + '=' + str(self._session_id)
}
response = requests.request(method=method,
timeout=settings.REQUEST_TIMEOUT,
url=url,
headers=headers,
data=uri)
log.info(1, 'Response: ' + response.text)
response.close()
return [response.status_code, response.text]
except requests.exceptions.HTTPError as e:
log.error(str(e))
return [400, str(e)]
except requests.exceptions.RequestException as e:
log.error(str(e))
return [400, str(e)]
def get_route(self):
"""
Queries the image streaming service for the route corresponding to the
current session
"""
log.info(1, 'Check if route already exists')
status = self.__do_request('GET', '')
if status[0] == 200:
log.info(1, 'Route exists: ' + str(status[1]))
return status
elif status[0] == 404:
# Create new route
log.error('Route does not exist for session ' + str(self._session_id) +
', creating it with ' + str(self.__get_uri()))
status = self.add_route()
if status[0] == 201:
return self.__do_request('GET', '')
else:
response = 'Image streaming service (' + settings.IMAGE_STREAMING_SERVICE_URL + \
') failed to create new route: ' + str(status[1])
log.error(response)
return [400, response]
else:
response = 'Image streaming service (' + settings.IMAGE_STREAMING_SERVICE_URL + \
') is unreachable: ' + str(status[1])
log.error(response)
return [400, response]
def get_route(self):
"""
Queries the image streaming service for the route corresponding to the
current session
"""
log.info(1, 'Check if route already exists')
status = self.__do_request('GET', '')
if status[0] == 200:
log.info(1, 'Route exists: ' + str(status[1]))
return status
elif status[0] == 404:
# Create new route
log.error('Route does not exist for session ' +
str(self._session_id) + ', creating it with ' +
str(self.__get_uri()))
status = self.add_route()
if status[0] == 201:
return self.__do_request('GET', '')
else:
response = 'Image streaming service (' + settings.IMAGE_STREAMING_SERVICE_URL + \
') failed to create new route: ' + str(status[1])
log.error(response)
return [400, response]
else:
response = 'Image streaming service (' + settings.IMAGE_STREAMING_SERVICE_URL + \
') is unreachable: ' + str(status[1])
log.error(response)
return [400, response]
def _query(session, attribute=None):
"""
Queries Slurm for information
:param session: Current user session
:param attribute: Attribute to be queried
:return: A Json response containing an ok status or a description of the error
"""
value = ''
if session.job_id is not None:
try:
command_line = SLURM_SSH_COMMAND + session.cluster_node + \
' scontrol show job ' + str(session.job_id)
process = subprocess.Popen(
[command_line],
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = process.communicate()[0]
if attribute is None:
return output
value = re.search(attribute + r'=(\w+)', output).group(1)
log.info(1, attribute + ' = ' + value)
return value
except OSError as e:
log.error(str(e))
return value
def stop(session_info):
"""
Gently stops a given process, waits for 2 seconds and checks for its disappearance
:param session_info: Session information containing the PID of the process
:return: A Json response containing on ok status or a description of the error
"""
if session_info.process_pid == -1:
return [404, 'Process does not exist']
try:
settings = \
manager.RenderingResourceSettings.objects.get(id=session_info.renderer_id)
if settings.graceful_exit:
try:
url = 'http://' + session_info.http_host + ':' + \
str(session_info.http_port) + '/' + 'EXIT'
req = urllib2.Request(url=url)
urllib2.urlopen(req).read()
# pylint: disable=W0702
except urllib2.URLError as e:
log.error('Cannot gracefully exit.' + str(e))
log.info(1, 'Terminating process ' + str(session_info.process_pid))
os.kill(session_info.process_pid, signal.SIGTERM)
time.sleep(2.0)
result = ProcessManager.__kill(session_info)
except OSError as e:
log.error(str(e))
result = [400, str(e)]
return result
def __forward_request(cls, session, command, request):
"""
Forwards the HTTP request to the rendering resource held by the given session
:param : session: Session holding the rendering resource
:param : command: Command passed to the rendering resource
:param : request: HTTP request
:rtype : An HTTP response containing the status and description of the command
"""
# query the status of the current session
status = cls.__session_status(session)
if status[0] != 200:
return HttpResponse(status=status[0], content=status[1])
try:
# Any other command is forwarded to the rendering resource
url = 'http://' + session.http_host + ':' + str(session.http_port) + '/' + command
log.info(1, 'Querying ' + str(url))
headers = tools.get_request_headers(request)
response = requests.request(
method=request.method, timeout=settings.REQUEST_TIMEOUT,
url=url, headers=headers, data=request.body)
data = response.content
response.close()
return HttpResponse(status=response.status_code, content=data)
except requests.exceptions.RequestException as e:
response = json.dumps({'contents': str(e)})
return HttpResponse(status=400, content=response)
def _rendering_resource_log(self, session, extension):
"""
Returns the contents of the specified file
:param session: Current user session
:param extension: File extension (typically err or out)
:return: A string containing the log
"""
try:
result = 'Not currently available'
if session.status in [SESSION_STATUS_STARTING, SESSION_STATUS_RUNNING]:
filename = self._file_name(session, extension)
command_line = SLURM_SSH_COMMAND + session.cluster_node + \
' cat ' + filename
log.info(1, 'Querying log: ' + command_line)
process = subprocess.Popen(
[command_line],
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
output = process.communicate()[0]
result = output
return result
except OSError as e:
return str(e)
except IOError as e:
return str(e)
def __forward_request(cls, session, command, request):
"""
Forwards the HTTP request to the rendering resource held by the given session
:param : session: Session holding the rendering resource
:param : command: Command passed to the rendering resource
:param : request: HTTP request
:rtype : An HTTP response containing the status and description of the command
"""
# query the status of the current session
status = cls.__session_status(session)
if status[0] != 200:
return HttpResponse(status=status[0], content=status[1])
try:
# Any other command is forwarded to the rendering resource
url = 'http://' + session.http_host + ':' + str(
session.http_port) + '/' + command
log.info(1, 'Querying ' + str(url))
headers = tools.get_request_headers(request)
response = requests.request(method=request.method,
timeout=settings.REQUEST_TIMEOUT,
url=url,
headers=headers,
data=request.body)
data = response.content
response.close()
return HttpResponse(status=response.status_code, content=data)
except requests.exceptions.RequestException as e:
response = json.dumps({'contents': str(e)})
return HttpResponse(status=400, content=response)
def __image_feed(cls, session_id):
"""
Get the route to image streaming server
:param : session_id: Id of the session holding the rendering resource
:rtype : An HTTP response containing uri of the image streaming server for the given session
"""
log.info(1, 'Requesting image feed')
ifm = image_feed_manager.ImageFeedManager(session_id)
return ifm.get_route()
def __image_feed(cls, session_id):
"""
Get the route to image streaming server
:param : session_id: Id of the session holding the rendering resource
:rtype : An HTTP response containing uri of the image streaming server for the given session
"""
log.info(1, 'Requesting image feed')
ifm = image_feed_manager.ImageFeedManager(session_id)
return ifm.get_route()
def get_session_id_from_request(request):
"""
Utility function that returns the session ID from a given HTTP request
:return: a UUID session identifier
"""
log.debug(1, 'Getting cookie from request')
try:
session_id = request.COOKIES[consts.COOKIE_ID]
except KeyError as e:
session_id = SessionManager.get_session_id()
log.info(1, 'Cookie ' + str(e) + ' is missing. New session ID is ' + str(session_id))
return session_id
def allocate(self, session, job_information):
"""
Allocates a job according to rendering resource configuration. If the allocation is
successful, the session job_id is populated and the session status is set to
SESSION_STATUS_SCHEDULED
:param session: Current user session
:param job_information: Information about the job
:return: A Json response containing on ok status or a description of the error
"""
status = None
for cluster_node in global_settings.SLURM_HOSTS:
try:
self._mutex.acquire()
session.status = SESSION_STATUS_SCHEDULING
session.cluster_node = cluster_node
session.save()
log.info(1, 'Scheduling job for session ' + session.id)
job_information.cluster_node = cluster_node
command_line = self._build_allocation_command(session, job_information)
process = subprocess.Popen(
[command_line],
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
error = process.communicate()[1]
if len(re.findall('Granted', error)) != 0:
session.job_id = re.findall('\\d+', error)[0]
log.info(1, 'Allocated job ' + str(session.job_id) +
' on cluster node ' + cluster_node)
session.status = SESSION_STATUS_SCHEDULED
session.save()
response = json.dumps({'message': 'Job scheduled', 'jobId': session.job_id})
status = [200, response]
break
else:
session.status = SESSION_STATUS_FAILED
session.save()
log.error(error)
response = json.dumps({'contents': error})
status = [400, response]
process.stdin.close()
except OSError as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
status = [400, response]
finally:
if self._mutex.locked():
self._mutex.release()
return status
def __kill(session_info):
"""
Kills a given process. This method should only be used if the stop method failed.
:param session_info: Session information containing the PID of the process
:return: A Json response containing on ok status or a description of the error
"""
try:
os.kill(session_info.process_pid, 0)
except OSError as e:
log.info(1, str('Failed to stop process. Killing it: ' + e.message))
os.kill(session_info.process_pid, signal.SIGKILL)
# Make sure any zombie child process is removed
os.waitpid(-1, os.WNOHANG)
return [200, 'Successfully closed process ' + str(session_info.process_pid)]
def _build_allocation_command(session, job_information):
"""
Builds the SLURM allocation command line
:param session: Current user session
:param job_information: Information about the job
:return: A string containing the SLURM command
"""
rr_settings = \
manager.RenderingResourceSettingsManager.get_by_id(session.renderer_id.lower())
options = ''
if job_information.exclusive_allocation or rr_settings.exclusive:
options += ' --exclusive'
value = rr_settings.nb_nodes
if job_information.nb_nodes != 0:
value = job_information.nb_nodes
if value != 0:
options += ' -N ' + str(value)
value = rr_settings.nb_cpus
if job_information.nb_cpus != 0:
value = job_information.nb_cpus
options += ' -c ' + str(value)
value = rr_settings.nb_gpus
if job_information.nb_gpus != 0:
value = job_information.nb_gpus
options += ' --gres=gpu:' + str(value)
if job_information.reservation != '' and job_information.reservation is not None:
options += ' --reservation=' + job_information.reservation
allocation_time = global_settings.SLURM_DEFAULT_TIME
if job_information.allocation_time != '':
allocation_time = job_information.allocation_time
log.info(1, 'Scheduling job for session ' + session.id)
job_name = session.owner + '_' + rr_settings.id
command_line = SLURM_SSH_COMMAND + session.cluster_node + \
' salloc --no-shell' + \
' --immediate=' + str(settings.SLURM_ALLOCATION_TIMEOUT) + \
' -p ' + rr_settings.queue + \
' --account=' + rr_settings.project + \
' --job-name=' + job_name + \
' --time=' + allocation_time + \
options
return command_line
def start(session_info, params, environment):
"""
Gently starts a given process, waits for 2 seconds and checks for its appearance
:param session_info: Session information containing the PID of the process
:return: A Json response containing on ok status or a description of the error
"""
try:
settings = manager.RenderingResourceSettingsManager.get_by_id(
session_info.renderer_id.lower())
default_parameters = manager.RenderingResourceSettingsManager.format_rest_parameters(
str(settings.process_rest_parameters_format),
str(session_info.http_host),
str(session_info.http_port),
'rest' + str(settings.id + session_info.id))
command_line = [
str(settings.command_line)
] + default_parameters.split()
try:
command_line += str(params).split()
except KeyError:
log.debug(1, 'No parameters specified')
except TypeError:
log.debug(1, 'No parameters specified')
environment_variables = settings.environment_variables.split() + environment.split(',')
process_env = os.environ.copy()
for environment_variable in environment_variables:
if environment_variable != '':
variable = environment_variable.split('=')
process_env[variable[0]] = variable[1]
log.info(1, 'Launching ' + settings.id + ' with ' + str(command_line))
process = subprocess.Popen(
command_line,
env=process_env,
shell=False,
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
session_info.process_pid = process.pid
session_info.status = SESSION_STATUS_STARTING
response = json.dumps(
{'message': 'Process started', 'processId': str(session_info.process_pid)})
return [200, response]
except RenderingResourceSettings.DoesNotExist as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
return [404, response]
def create_session(cls, request):
"""
Creates a user session
:param : request: request containing the launching parameters of the rendering resource
:rtype : An HTTP response containing on ok status or a description of the error
"""
sm = session_manager.SessionManager()
# Create new Cookie ID for new session
session_id = sm.get_session_id()
try:
status = sm.create_session(
session_id, request.DATA['owner'], request.DATA['renderer_id'])
response = HttpResponse(status=status[0], content=status[1])
response.set_cookie(consts.COOKIE_ID, session_id)
log.info(1, 'Session created ' + str(session_id))
return response
except KeyError as e:
return HttpResponse(status=401, content=str(e))
else:
return HttpResponse(status=401, content='Unexpected exception')
def create_session(cls, request):
"""
Creates a user session
:param : request: request containing the launching parameters of the rendering resource
:rtype : An HTTP response containing on ok status or a description of the error
"""
sm = session_manager.SessionManager()
# Create new session ID
session_id = sm.get_session_id()
try:
status = sm.create_session(session_id, request.DATA['owner'],
request.DATA['renderer_id'])
response = HttpResponse(status=status[0],
content=json.dumps(
{'session_id': str(session_id)}))
log.info(1, 'Session created ' + str(session_id))
return response
except KeyError as e:
return HttpResponse(status=401, content=str(e))
else:
return HttpResponse(status=401, content='Unexpected exception')
def request_vocabulary(session_id):
"""
Queries the rendering resource vocabulary
:param session_id: Id of the session to be queried
:return 200 code if rendering resource is able to provide vocabulary. 503
otherwise. 404 if specified session does not exist.
"""
try:
session = Session.objects.get(id=session_id)
try:
url = 'http://' + session.http_host + ':' + \
str(session.http_port) + '/' + consts.RR_SPECIFIC_COMMAND_VOCABULARY
log.info(1, 'Requesting vocabulary from ' + url)
r = requests.put(url=url,
timeout=global_settings.REQUEST_TIMEOUT)
response = r.text
r.close()
return [http_status.HTTP_200_OK, response]
except requests.exceptions.RequestException as e:
log.info(1, str(e))
return [http_status.HTTP_503_SERVICE_UNAVAILABLE, str(e)]
except Session.DoesNotExist as e:
# Requested session does not exist
log.info(1, str(e))
return [http_status.HTTP_404_NOT_FOUND, str(e)]
def run(self):
"""
Checks for active sessions. If no keep-alive message was received in the last
n seconds, the session is closed.
"""
while self.signal:
log.info(1, 'Checking for inactive sessions')
for session in self.sessions.all():
log.info(
1, 'Session ' + str(session.id) + ' is valid until ' +
str(session.valid_until))
if datetime.datetime.now() > session.valid_until:
log.info(
1, "Session " + str(session.id) +
" timed out. Session will now be closed")
session.status = SESSION_STATUS_STOPPING
with transaction.atomic():
session.save()
if session.process_pid != -1:
process_manager.ProcessManager.stop(session)
if session.job_id is not None and session.job_id != '':
job_manager.globalJobManager.stop(session)
with transaction.atomic():
session.delete()
time.sleep(KEEP_ALIVE_FREQUENCY)
def delete_session(cls, session_id):
"""
Deletes a session
:param session_id: Id of the session to delete
:rtype A tuple containing the status and a potential error description
"""
try:
session = Session.objects.get(id=session_id)
if not session.status == SESSION_STATUS_STOPPING:
log.info(1, 'Removing session ' + str(session_id))
session.status = SESSION_STATUS_STOPPING
session.save()
if session.process_pid != -1:
process_manager.ProcessManager.stop(session)
if session.job_id is not None and session.job_id != '':
jm = job_manager.JobManager()
jm.stop(session)
session.delete()
msg = 'Session successfully destroyed'
log.info(1, msg)
response = json.dumps({'contents': str(msg)})
return [http_status.HTTP_200_OK, response]
except Session.DoesNotExist as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
return [http_status.HTTP_404_NOT_FOUND, response]
except Exception as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
return [http_status.HTTP_500_INTERNAL_ERROR, response]
msg = 'Session is currently being destroyed'
response = json.dumps({'contents': msg})
log.info(1, msg)
return [http_status.HTTP_200_OK, response]
def kill(session):
"""
Kills the given job. This method should only be used if the stop method failed.
:param session: Current user session
:return: A Json response containing on ok status or a description of the error
"""
result = [500, 'Unexpected error']
if session.job_id is not None:
try:
command_line = SLURM_SSH_COMMAND + session.cluster_node + \
' scancel ' + session.job_id
log.info(1, 'Stopping job ' + session.job_id)
process = subprocess.Popen(
[command_line],
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
output = process.communicate()[0]
log.info(1, output)
msg = 'Job successfully cancelled'
log.info(1, msg)
response = json.dumps({'contents': msg})
result = [200, response]
except OSError as e:
msg = str(e)
log.error(msg)
response = json.dumps({'contents': msg})
result = [400, response]
return result
def request_vocabulary(session_id):
"""
Queries the rendering resource vocabulary
:param session_id: Id of the session to be queried
:return 200 code if rendering resource is able to provide vocabulary. 503
otherwise. 404 if specified session does not exist.
"""
try:
session = Session.objects.get(id=session_id)
try:
url = 'http://' + session.http_host + ':' + \
str(session.http_port) + '/' + consts.RR_SPECIFIC_COMMAND_VOCABULARY
log.info(1, 'Requesting vocabulary from ' + url)
r = requests.put(
url=url,
timeout=global_settings.REQUEST_TIMEOUT)
response = r.text
r.close()
return [http_status.HTTP_200_OK, response]
except requests.exceptions.RequestException as e:
log.info(1, str(e))
return [http_status.HTTP_503_SERVICE_UNAVAILABLE, str(e)]
except Session.DoesNotExist as e:
# Requested session does not exist
log.info(1, str(e))
return [http_status.HTTP_404_NOT_FOUND, str(e)]
def delete_session(cls, session_id):
"""
Deletes a session
:param session_id: Id of the session to delete
:rtype A tuple containing the status and a potential error description
"""
try:
session = Session.objects.get(id=session_id)
if not session.status == SESSION_STATUS_STOPPING:
log.info(1, 'Removing session ' + str(session_id))
session.status = SESSION_STATUS_STOPPING
session.save()
if session.process_pid != -1:
process_manager.ProcessManager.stop(session)
if session.job_id is not None and session.job_id != '':
jm = job_manager.JobManager()
jm.stop(session)
session.delete()
msg = 'Session successfully destroyed'
log.info(1, msg)
response = json.dumps({'contents': str(msg)})
return [http_status.HTTP_200_OK, response]
except Session.DoesNotExist as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
return [http_status.HTTP_404_NOT_FOUND, response]
except Exception as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
return [http_status.HTTP_500_INTERNAL_ERROR, response]
msg = 'Session is currently being destroyed'
response = json.dumps({'contents': msg})
log.info(1, msg)
return [http_status.HTTP_200_OK, response]
def stop(self, session):
"""
Gently stops a given job, waits for 2 seconds and checks for its disappearance
:param session: Current user session
:return: A Json response containing on ok status or a description of the error
"""
result = [500, 'Unexpected error']
try:
self._mutex.acquire()
# pylint: disable=E1101
setting = \
manager.RenderingResourceSettings.objects.get(
id=session.renderer_id)
if setting.graceful_exit:
log.info(1, 'Gracefully exiting rendering resource')
try:
url = 'http://' + session.http_host + \
':' + str(session.http_port) + '/' + \
settings.RR_SPECIFIC_COMMAND_EXIT
log.info(1, url)
r = requests.put(
url=url,
timeout=global_settings.REQUEST_TIMEOUT)
r.close()
# pylint: disable=W0702
except requests.exceptions.RequestException as e:
log.error(traceback.format_exc(e))
result = self.kill(session)
except OSError as e:
msg = str(e)
log.error(msg)
response = json.dumps({'contents': msg})
result = [400, response]
finally:
if self._mutex.locked():
self._mutex.release()
return result
def __do_request(self, method, uri):
"""
Creates an HTTP request and invokes the image streaming service
:param method Method to be used by the HTTP request (GET, POST, DELETE, etc)
:param uri JSON formatted URI to attach to the HTTP request
:return 200 if successful, Error code and message otherwise
"""
try:
url = settings.IMAGE_STREAMING_SERVICE_URL + '/route'
log.info(1, '__do_request(' + method + ', ' + url + ')')
headers = {'Content-Type': 'application/json',
'Cookie': COOKIE_ID + '=' + str(self._session_id)}
response = requests.request(
method=method, timeout=settings.REQUEST_TIMEOUT,
url=url, headers=headers, data=uri)
log.info(1, 'Response: ' + response.text)
response.close()
return [response.status_code, response.text]
except requests.exceptions.HTTPError as e:
log.error(str(e))
return [400, str(e)]
except requests.exceptions.RequestException as e:
log.error(str(e))
return [400, str(e)]
def destroy_session(cls, request):
"""
Stops the renderer and destroys the user session
:param : request: The REST request
:rtype : An HTTP response containing on ok status or a description of the error
"""
sm = session_manager.SessionManager()
session_id = sm.get_session_id_from_request(request)
log.info(1, 'Remove image feed route if it exists')
ifm = image_feed_manager.ImageFeedManager(session_id)
status = ifm.remove_route()
if status[0] != 200:
log.error(status[1])
log.info(1, 'Remove session from db')
status = sm.delete_session(session_id)
log.info(1, 'Session deleted ' + str(session_id))
return HttpResponse(status=status[0], content=status[1])
def destroy_session(cls, request):
"""
Stops the renderer and destroys the user session
:param : request: The REST request
:rtype : An HTTP response containing on ok status or a description of the error
"""
sm = session_manager.SessionManager()
session_id = sm.get_session_id_from_request(request)
log.info(1, 'Remove image feed route if it exists')
ifm = image_feed_manager.ImageFeedManager(session_id)
status = ifm.remove_route()
if status[0] != 200:
log.error(status[1])
log.info(1, 'Remove session from db')
status = sm.delete_session(session_id)
log.info(1, 'Session deleted ' + str(session_id))
return HttpResponse(status=status[0], content=status[1])
def __verify_hostname(cls, session):
"""
Verify the existence of an hostname for the current session, and tries
to populate it if null
:param : session: Session holding the rendering resource
"""
log.info(
1, 'Verifying hostname ' + session.http_host + ' for session ' +
str(session.id))
if not session.status == SESSION_STATUS_GETTING_HOSTNAME and \
session.job_id and session.http_host == '':
session.status = SESSION_STATUS_GETTING_HOSTNAME
session.save()
log.info(
1, 'Querying JOB hostname for job id: ' + str(session.job_id))
hostname = job_manager.globalJobManager.hostname(session)
if hostname == '':
msg = 'Job scheduled but ' + session.renderer_id + ' is not yet running'
log.error(msg)
session.status = SESSION_STATUS_SCHEDULED
session.save()
response = json.dumps({'contents': str(msg)})
return [404, response]
elif hostname == 'FAILED':
sm = session_manager.SessionManager()
sm.delete_session(session.id)
msg = 'Job as been cancelled'
log.error(msg)
response = json.dumps({'contents': str(msg)})
return [404, response]
else:
session.http_host = hostname
session.save()
msg = 'Resolved hostname for job ' + str(session.job_id) + ' to ' + \
str(session.http_host)
log.info(1, msg)
response = json.dumps({'contents': str(msg)})
return [200, response]
response = json.dumps(
{'contents': str('Job is running on host ' + session.http_host)})
return [200, response]
def __verify_hostname(cls, session):
"""
Verify the existence of an hostname for the current session, and tries
to populate it if null
:param : session: Session holding the rendering resource
"""
log.info(1, 'Verifying hostname ' + session.http_host + ' for session ' + str(session.id))
if not session.status == SESSION_STATUS_GETTING_HOSTNAME and \
session.job_id and session.http_host == '':
session.status = SESSION_STATUS_GETTING_HOSTNAME
session.save()
log.info(1, 'Querying JOB hostname for job id: ' + str(session.job_id))
hostname = job_manager.globalJobManager.hostname(session.job_id)
if hostname == '':
msg = 'Job scheduled but ' + session.renderer_id + ' is not yet running'
log.error(msg)
session.status = SESSION_STATUS_SCHEDULED
session.save()
response = json.dumps({'contents': str(msg)})
return [404, response]
elif hostname == 'FAILED':
sm = session_manager.SessionManager()
sm.delete_session(session.id)
msg = 'Job as been cancelled'
log.error(msg)
response = json.dumps({'contents': str(msg)})
return [404, response]
else:
session.http_host = hostname
session.save()
msg = 'Resolved hostname for job ' + str(session.job_id) + ' to ' + \
str(session.http_host)
log.info(1, msg)
response = json.dumps({'contents': str(msg)})
return [200, response]
response = json.dumps({'contents': str('Job is running on host ' + session.http_host)})
return [200, response]
def __init__(self, sessions):
threading.Thread.__init__(self)
self.signal = True
self.sessions = sessions
log.info(1, "Keep-Alive thread started...")
def start(self, session, job_information):
"""
Start the rendering resource using the job allocated by the schedule method. If successful,
the session status is set to SESSION_STATUS_STARTING
:param session: Current user session
:param job_information: Information about the job
:return: A Json response containing on ok status or a description of the error
"""
try:
self._mutex.acquire()
session.status = SESSION_STATUS_STARTING
session.save()
rr_settings = \
manager.RenderingResourceSettingsManager.get_by_id(session.renderer_id.lower())
# Modules
full_command = 'module purge\n'
values = rr_settings.modules.split()
for module in values:
full_command += 'module load ' + module.strip() + '\n'
# Environment variables
values = rr_settings.environment_variables.split()
values += job_information.environment.split()
for variable in values:
full_command += variable + ' '
# Command lines parameters
rest_parameters = manager.RenderingResourceSettingsManager.format_rest_parameters(
str(rr_settings.scheduler_rest_parameters_format),
str(session.http_host),
str(session.http_port),
'rest' + str(rr_settings.id + session.id))
values = rest_parameters.split()
values += job_information.params.split()
full_command += rr_settings.command_line
for parameter in values:
full_command += ' ' + parameter
# Output redirection
full_command += ' > ' + self._file_name(session, settings.SLURM_OUT_FILE)
full_command += ' 2> ' + self._file_name(session, settings.SLURM_ERR_FILE)
full_command += ' &\n'
# Start Process on cluster
command_line = '/usr/bin/ssh -i ' + \
global_settings.SLURM_SSH_KEY + ' ' + \
global_settings.SLURM_USERNAME + '@' + \
session.http_host
log.info(1, 'Connect to cluster machine: ' + command_line)
process = subprocess.Popen(
[command_line],
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.info(1, 'Full command:\n' + full_command)
process.stdin.write(full_command)
output = process.communicate()[0]
log.info(1, output)
process.stdin.close()
session.status = SESSION_STATUS_STARTING
session.save()
response = json.dumps({'message': session.renderer_id + ' successfully started'})
return [200, response]
except OSError as e:
log.error(str(e))
response = json.dumps({'contents': str(e)})
return [400, response]
finally:
if self._mutex.locked():
self._mutex.release()
def query_status(session_id):
"""
Queries the session status and updates it accordingly
- Stopped: Default status, when no rendering resource is active
- Scheduled: The slurm job was created but the rendering resource is not yet started.
- Starting: The rendering resource is started but is not ready to respond to REST requests
- Running: The rendering resource is started and ready to respond to REST requests
- Stopping: tThe request for stopping the slurm job was made, but the application is not yet
terminated
:param session_id: Id of the session to be queried
:return 200 code if rendering resource is able to process REST requests. 503
otherwise. 404 if specified session does not exist.
"""
try:
session = Session.objects.get(id=session_id)
status_description = 'Undefined'
session_status = session.status
log.info(1, 'Current session status is: ' + str(session_status))
if session_status == SESSION_STATUS_SCHEDULING:
status_description = str(session.renderer_id + ' is scheduled')
elif session_status == SESSION_STATUS_SCHEDULED or \
session_status == SESSION_STATUS_GETTING_HOSTNAME:
if session.http_host != '':
status_description = session.renderer_id + ' is starting'
log.info(1, status_description)
session.status = SESSION_STATUS_STARTING
session.save()
else:
status_description = str(session.renderer_id + ' is scheduled')
elif session_status == SESSION_STATUS_STARTING:
# Rendering resource might be running but not yet capable of
# serving REST requests. The vocabulary is invoked to make
# sure that the rendering resource is ready to serve REST
# requests.
rr_settings = \
manager.RenderingResourceSettingsManager.get_by_id(session.renderer_id.lower())
if not rr_settings.wait_until_running:
status_description = session.renderer_id + ' is up and running'
log.info(1, status_description)
session.status = SESSION_STATUS_RUNNING
session.save()
else:
log.info(1, 'Requesting rendering resource vocabulary')
status = SessionManager.request_vocabulary(session_id)
if status[0] == http_status.HTTP_200_OK:
status_description = session.renderer_id + ' is up and running'
log.info(1, status_description)
session.status = SESSION_STATUS_RUNNING
session.save()
else:
status_description = session.renderer_id + \
' is starting but the HTTP interface is not yet available'
elif session_status == SESSION_STATUS_RUNNING:
# Rendering resource is currently running
status_description = session.renderer_id + ' is up and running'
# Update the timestamp if the current value is expired
sgs = SystemGlobalSettings.objects.get()
if datetime.datetime.now() > session.valid_until:
session.valid_until = datetime.datetime.now() + datetime.timedelta(
seconds=sgs.session_keep_alive_timeout)
session.save()
elif session_status == SESSION_STATUS_STOPPING:
# Rendering resource is currently in the process of terminating.
status_description = str(session.renderer_id + ' is terminating...')
session.delete()
session.save()
elif session_status == SESSION_STATUS_STOPPED:
# Rendering resource is currently not active.
status_description = str(session.renderer_id + ' is not active')
elif session_status == SESSION_STATUS_FAILED:
status_description = str('Job allocation failed for ' + session.renderer_id)
status_code = session.status
response = [http_status.HTTP_200_OK, json.dumps({
'session': str(session_id),
'code': status_code,
'description': status_description,
'hostname': session.http_host,
'port': str(session.http_port),
})]
return response
except Session.DoesNotExist as e:
# Requested session does not exist
log.error(str(e))
return [http_status.HTTP_404_NOT_FOUND, str(e)]
def query_status(session_id):
"""
Queries the session status and updates it accordingly
- Stopped: Default status, when no rendering resource is active
- Scheduled: The slurm job was created but the rendering resource is not yet started.
- Starting: The rendering resource is started but is not ready to respond to REST requests
- Running: The rendering resource is started and ready to respond to REST requests
- Stopping: tThe request for stopping the slurm job was made, but the application is not yet
terminated
:param session_id: Id of the session to be queried
:return 200 code if rendering resource is able to process REST requests. 503
otherwise. 404 if specified session does not exist.
"""
try:
session = Session.objects.get(id=session_id)
status_description = 'Undefined'
session_status = session.status
log.info(1, 'Current session status is: ' + str(session_status))
if session_status == SESSION_STATUS_SCHEDULING:
status_description = str(session.renderer_id + ' is scheduled')
elif session_status == SESSION_STATUS_SCHEDULED or \
session_status == SESSION_STATUS_GETTING_HOSTNAME:
if session.http_host != '':
status_description = session.renderer_id + ' is starting'
log.info(1, status_description)
session.status = SESSION_STATUS_STARTING
session.save()
else:
status_description = str(session.renderer_id +
' is scheduled')
elif session_status == SESSION_STATUS_STARTING:
# Rendering resource might be running but not yet capable of
# serving REST requests. The vocabulary is invoked to make
# sure that the rendering resource is ready to serve REST
# requests.
rr_settings = \
manager.RenderingResourceSettingsManager.get_by_id(session.renderer_id.lower())
if not rr_settings.wait_until_running:
status_description = session.renderer_id + ' is up and running'
log.info(1, status_description)
session.status = SESSION_STATUS_RUNNING
session.save()
else:
log.info(1, 'Requesting rendering resource vocabulary')
status = SessionManager.request_vocabulary(session_id)
if status[0] == http_status.HTTP_200_OK:
status_description = session.renderer_id + ' is up and running'
log.info(1, status_description)
session.status = SESSION_STATUS_RUNNING
session.save()
else:
status_description = session.renderer_id + \
' is starting but the HTTP interface is not yet available'
elif session_status == SESSION_STATUS_RUNNING:
# Rendering resource is currently running
status_description = session.renderer_id + ' is up and running'
# Update the timestamp if the current value is expired
sgs = SystemGlobalSettings.objects.get()
if datetime.datetime.now() > session.valid_until:
session.valid_until = datetime.datetime.now(
) + datetime.timedelta(
seconds=sgs.session_keep_alive_timeout)
session.save()
elif session_status == SESSION_STATUS_STOPPING:
# Rendering resource is currently in the process of terminating.
status_description = str(session.renderer_id +
' is terminating...')
session.delete()
session.save()
elif session_status == SESSION_STATUS_STOPPED:
# Rendering resource is currently not active.
status_description = str(session.renderer_id +
' is not active')
elif session_status == SESSION_STATUS_FAILED:
status_description = str('Job allocation failed for ' +
session.renderer_id)
status_code = session.status
response = [
http_status.HTTP_200_OK,
json.dumps({
'session': str(session_id),
'code': status_code,
'description': status_description,
'hostname': session.http_host,
'port': str(session.http_port),
})
]
return response
except Session.DoesNotExist as e:
# Requested session does not exist
log.error(str(e))
return [http_status.HTTP_404_NOT_FOUND, str(e)]
