def on_put(self, req, res):
"""
PUT: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
raw = str(req.stream.read(), 'UTF-8')
data = eval(raw)
LookUpTable.update_worker(data)
SysOut.debug_string("Update worker status ({0})".format(
data[Definition.get_str_node_name()]))
res.body = "Okay"
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
format_response_string(res, falcon.HTTP_401, "Token is required")
return
if req.params[Definition.get_str_token()] == Setting.get_token():
result = LService.get_machine_status(Setting, CRole.MASTER)
format_response_string(res, falcon.HTTP_200, str(result))
else:
format_response_string(res, falcon.HTTP_401, "Invalid token ID")
def on_post(self, req, res):
# check token and request type is provided
req_raw = (str(req.stream.read(req.content_length or 0),
'utf-8')) # create dict of body data if they exist
req_data = json.loads(req_raw)
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not "type" in req.params:
res.body = "No command specified."
res.content_type = "String"
res.status = falcon.HTTP_406
return
# request to create new job - create ID for job, add to lookup table, queue creation of the job
if req.params['type'] == 'new_job':
job = new_job(
req_data) # attempt to create new job from provided parameters
if not job:
SysOut.err_string("New job could not be added!")
format_response_string(res, falcon.HTTP_500,
"Could not create job.")
return
job_status = job.get('job_status')
format_response_string(
res, falcon.HTTP_200,
"Job request received, container status: {}\nJob ID: {}".
format(job_status, job.get('job_id')))
return
return
def on_get(self, req, res):
# check token and request type is provided
if not Definition.get_str_token() in req.params:
format_response_string(res, falcon.HTTP_401, "Token required.")
return
if not "type" in req.params:
format_response_string(res, falcon.HTTP_406,
"Command not specified.")
return
# user wants to know if containers are ready for provided job ID
if req.params['type'] == "poll_job":
id = req.params.get('job_id')
if not id in LookUpTable.Jobs.verbose():
format_response_string(res, falcon.HTTP_404,
"Specified job not available.")
return
jobs = LookUpTable.Jobs.verbose()
stat = str(jobs[id].get('job_status'))
format_response_string(res, falcon.HTTP_200,
("Job status: " + stat))
return
def on_put(self, req, res):
"""
PUT: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if Definition.Docker.get_str_finished() in req.params:
# a container is shutting down, update containers
# TODO: add some kind of safety mechanism to really make sure no new requests have been sent to this container before acknowledging removal?
if LookUpTable.remove_container(
req.params.get(
Definition.Container.get_str_con_image_name()),
req.params.get(Definition.Docker.get_str_finished())):
format_response_string(res, falcon.HTTP_200,
"Container successfully removed")
# NOTE: container will terminate as soon as it reads this response!
else:
format_response_string(
res, falcon.HTTP_400,
"Could not remove container from table!")
# NOTE: container will continue as before when it reads this response!
return
if req.params[Definition.get_str_token()] == Setting.get_token():
data = json.loads(
str(req.stream.read(req.content_length or 0), 'utf-8'))
LookUpTable.update_worker(data)
SysOut.debug_string("Update worker status ({0})".format(
data[Definition.get_str_node_name()]))
res.body = "Okay"
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
return
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
result = LService.get_machine_status(Setting, CRole.MASTER)
res.body = str(result)
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def on_get(self, req, res):
"""
GET: /messagesQuery?token=None&command=queueLength
This function inquiry about the number of messages in queue. For dealing with create a new instance.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.MessagesQueue.get_str_command() in req.params:
res.body = "No command specified."
res.content_type = "String"
res.status = falcon.HTTP_406
return
if req.params[Definition.MessagesQueue.get_str_command(
)] == Definition.MessagesQueue.get_str_queue_length():
res.body = str(MessagesQueue.get_queues_all())
res.content_type = "String"
res.status = falcon.HTTP_200
return
if req.params[Definition.MessagesQueue.get_str_command(
)] == Definition.MessagesQueue.get_str_current_id():
res.body = "None"
res.content_type = "String"
res.status = falcon.HTTP_200
return
if req.params[Definition.MessagesQueue.get_str_command()] == "verbose":
data = LookUpTable.verbose()
data['MSG'] = MessagesQueue.verbose()
if req.params.get('format') == 'JSON':
data = json.dumps(data)
res.body = str(data)
res.content_type = "String"
res.status = falcon.HTTP_200
if req.params[
Definition.MessagesQueue.get_str_command()] == "verbose_html":
data = LookUpTable.verbose()
data['MSG'] = MessagesQueue.verbose()
res.body = get_html_form(data['WORKERS'], data['MSG'],
data['CONTAINERS'], data['TUPLES'])
res.content_type = "String"
res.status = falcon.HTTP_200
def on_get(self, req, res):
"""
GET: /status?token={None}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if req.params[Definition.get_str_token()] == Setting.get_token():
s_content = Services.get_machine_status(Setting, CRole.WORKER)
s_content[Definition.REST.get_str_docker(
)] = DockerService.get_containers_status()
res.body = str(s_content)
res.content_type = "String"
res.status = falcon.HTTP_200
else:
res.body = "Invalid token ID."
res.content_type = "String"
res.status = falcon.HTTP_401
def update_job(request):
job_id = request.get('job_id')
if not job_id in LookUpTable.Jobs.__jobs:
SysOut.warn_string(
"Couldn't update job, no existing job matching ID!")
return False
tkn = request.get(Definition.get_str_token())
if not tkn == LookUpTable.Jobs.__jobs[job_id]['user_token']:
SysOut.warn_string("Incorrect token, refusing update.")
return False
old_job = LookUpTable.Jobs.__jobs[job_id]
old_job['job_status'] = request.get('job_status')
return True
def on_post(self, req, res):
"""
POST: docker?token=None&command={command}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.Docker.get_str_command() in req.params:
res.body = "Command is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
"""
POST: docker?token=None&command=create
"""
if req.params[Definition.Docker.get_str_command(
)] == Definition.Docker.get_str_create():
# Unpack the posted data
raw = str(req.stream.read(), 'UTF-8')
data = eval(raw)
if not data[Definition.Container.get_str_con_image_name()]:
res.body = "Required parameters are not supplied!"
res.content_type = "String"
res.status = falcon.HTTP_401
result = DockerService.create_container(
data[Definition.Container.get_str_con_image_name()])
if result:
res.body = "Okay"
res.content_type = "String"
res.status = falcon.HTTP_200
return
else:
res.body = "Create container error!"
res.content_type = "String"
res.status = falcon.HTTP_400
def new_job(request):
new_item = {}
new_id = request.get('job_id')
if not new_id:
SysOut.warn_string("Couldn't create job, no ID provided!")
return False
if new_id in LookUpTable.Jobs.__jobs:
SysOut.warn_string(
"Job already exists in system, can't create!")
return False
new_item['job_id'] = new_id
new_item['job_status'] = request.get('job_status')
new_item[
Definition.Container.get_str_con_image_name()] = request.get(
Definition.Container.get_str_con_image_name())
new_item['user_token'] = request.get(Definition.get_str_token())
new_item['volatile'] = request.get('volatile')
LookUpTable.Jobs.__jobs[new_id] = new_item
return True
def on_get(self, req, res):
"""
GET: docker?token=None&command={command}
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.Docker.get_str_command() in req.params:
res.body = "Command is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Check for status command
if req.params[Definition.Docker.get_str_command(
)] == Definition.Docker.get_str_status():
body = DockerService.get_containers_status()
res.body = str(body)
res.content_type = "String"
res.status = falcon.HTTP_200
def on_get(self, req, res):
"""
return "&c_name=" + container_name + "&c_os=" + container_os + "&priority=" + str(priority)
GET: /streamRequest?token=None
This function is mainly respond with the available channel for streaming from data source.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Check for required parameter.
if not Definition.Container.get_str_con_image_name() in req.params:
res.body = "Container name is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.Container.get_str_container_os() in req.params:
res.body = "Container os is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
if not Definition.Container.get_str_data_source() in req.params:
res.body = "Data digest is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Parse to dict object
ret = LookUpTable.Tuples.get_tuple_object(req)
if Definition.Container.get_str_container_priority() in req.params:
if LService.is_str_is_digit(req.params[
Definition.Container.get_str_container_priority()]):
ret[Definition.Container.get_str_container_priority()] = int(
req.params[
Definition.Container.get_str_container_priority()])
else:
res.body = "Container priority is not digit."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Register item into tuples
LookUpTable.Tuples.add_tuple_info(ret)
# Check for the availability of the container
ret = LookUpTable.get_candidate_container(
ret[Definition.Container.get_str_con_image_name()])
if ret:
res.body = Definition.Master.get_str_end_point(ret)
res.content_type = "String"
res.status = falcon.HTTP_200
return
else:
# No streaming end-point available
res.body = Definition.Master.get_str_end_point_MS(Setting)
res.content_type = "String"
res.status = falcon.HTTP_200
return
def on_post(self, req, res):
"""
POST: /streamRequest?token=None
This function invoked by the driver in micro-batch in the container.
It responds with getting a stream from data source or from messaging system.
"""
if not Definition.get_str_token() in req.params:
res.body = "Token is required."
res.content_type = "String"
res.status = falcon.HTTP_401
return
# Check that the PE is existing or not, if not insert and respond
if Definition.REST.Batch.get_str_batch_addr() in req.params and \
Definition.REST.Batch.get_str_batch_port() in req.params and \
Definition.REST.Batch.get_str_batch_status() in req.params and \
Definition.Container.get_str_con_image_name() in req.params:
# Check for data type
if req.params[Definition.REST.Batch.get_str_batch_port()].isdigit() and \
req.params[Definition.REST.Batch.get_str_batch_status()].isdigit():
ret = LookUpTable.Containers.get_container_object(req)
# If queue contain data, ignore update and stream from queue
length = MessagesQueue.get_queues_length(
ret[Definition.Container.get_str_con_image_name()])
if not length:
LookUpTable.Containers.update_container(ret)
SysOut.debug_string("No item in queue!")
res.body = "No item in queue"
res.content_type = "String"
res.status = falcon.HTTP_200
return
if length > 0 and ret[
Definition.REST.Batch.get_str_batch_status(
)] == CStatus.AVAILABLE:
# ret[Definition.REST.Batch.get_str_batch_status()] = CStatus.BUSY
# LookUpTable.Containers.update_container(ret)
res.data = bytes(
MessagesQueue.pop_queue(ret[
Definition.Container.get_str_con_image_name()]))
res.content_type = "Bytes"
res.status = falcon.HTTP_203
return
else:
# Register a new channel
LookUpTable.Containers.update_container(ret)
res.body = "OK"
res.content_type = "String"
res.status = falcon.HTTP_200
return
else:
res.body = "Invalid data type!"
res.content_type = "String"
res.status = falcon.HTTP_406
return
else:
res.body = "Invalid parameters!"
res.content_type = "String"
res.status = falcon.HTTP_406
return