def poll():
""" Callback function that polls for new tasks based on a schedule. """
deployment_id = helper.get_deployment_id()
# If the deployment is not registered, skip.
if not deployment_id:
return
# If we can't reach the backup and recovery services, skip.
nodes = helper.get_node_info()
http_client = tornado.httpclient.HTTPClient()
for node in nodes:
br_host = node[helper.NodeInfoTags.HOST]
request = tornado.httpclient.HTTPRequest(br_host)
try:
response = http_client.fetch(request)
if json.loads(response.body)['status'] != 'up':
logging.warn(
'Backup and Recovery service at {} is not up.'.format(
br_host))
return
except (socket.error, ValueError):
logging.exception(
'Backup and Recovery service at {} is not up.'.format(br_host))
return
logging.info("Polling for new task.")
# Send request to AppScale Portal.
url = "{0}{1}".format(constants.PORTAL_URL, constants.PORTAL_POLL_PATH)
data = urllib.urlencode({JSONTags.DEPLOYMENT_ID: deployment_id})
request = helper.create_request(url=url, method='POST', body=data)
response = helper.urlfetch(request)
if not response[JSONTags.SUCCESS]:
logging.error("Inaccessible resource: {}".format(url))
return
try:
data = json.loads(response[JSONTags.BODY])
except (TypeError, ValueError) as error:
logging.error(
"Cannot parse response from url '{0}'. Error: {1}".format(
url, str(error)))
return
if data == {}: # If there's no task to perform.
return
# Verify all necessary fields are present in the request.
if not set(data.keys()).issuperset(set(constants.REQUIRED_KEYS)):
logging.error("Missing args in response: {0}".format(response))
return
logging.debug("Task to run: {0}".format(data))
logging.info("Redirecting task request to TaskHandler.")
url = "http://localhost:{}{}".format(constants.HERMES_PORT, '/do_task')
request = helper.create_request(url, method='POST', body=json.dumps(data))
# The poller can move forward without waiting for a response here.
helper.urlfetch_async(request)
def poll():
""" Callback function that polls for new tasks based on a schedule. """
deployment_id = helper.get_deployment_id()
# If the deployment is not registered, skip.
if not deployment_id:
return
# If we can't reach the backup and recovery services, skip.
nodes = helper.get_node_info()
http_client = tornado.httpclient.HTTPClient()
for node in nodes:
br_host = node[helper.NodeInfoTags.HOST]
request = tornado.httpclient.HTTPRequest(br_host)
try:
response = http_client.fetch(request)
if json.loads(response.body)['status'] != 'up':
logging.warn('Backup and Recovery service at {} is not up.'
.format(br_host))
return
except (socket.error, ValueError):
logging.exception('Backup and Recovery service at {} is not up.'
.format(br_host))
return
logging.info("Polling for new task.")
# Send request to AppScale Portal.
url = "{0}{1}".format(constants.PORTAL_URL,
constants.PORTAL_POLL_PATH)
data = urllib.urlencode({JSONTags.DEPLOYMENT_ID: deployment_id})
request = helper.create_request(url=url, method='POST', body=data)
response = helper.urlfetch(request)
if not response[JSONTags.SUCCESS]:
logging.error("Inaccessible resource: {}".format(url))
return
try:
data = json.loads(response[JSONTags.BODY])
except (TypeError, ValueError) as error:
logging.error("Cannot parse response from url '{0}'. Error: {1}".
format(url, str(error)))
return
if data == {}: # If there's no task to perform.
return
# Verify all necessary fields are present in the request.
if not set(data.keys()).issuperset(set(constants.REQUIRED_KEYS)):
logging.error("Missing args in response: {0}".format(response))
return
logging.debug("Task to run: {0}".format(data))
logging.info("Redirecting task request to TaskHandler.")
url = "http://localhost:{}{}".format(constants.HERMES_PORT, '/do_task')
request = helper.create_request(url, method='POST', body=json.dumps(data))
# The poller can move forward without waiting for a response here.
helper.urlfetch_async(request)
def send(self, nodes_stats):
deployment_id = helper.get_deployment_id()
# If the deployment is not registered, skip.
if not deployment_id:
return
# Send request to AppScale Portal.
portal_path = self._portal_method.format(deployment_id=deployment_id)
url = "{0}{1}".format(constants.PORTAL_URL, portal_path)
data = {
'deployment_id':
deployment_id,
'nodes_stats':
json.dumps({
node_ip: [stats_to_dict(snapshot) for snapshot in snapshots]
for node_ip, snapshots in nodes_stats.iteritems()
})
}
snapshots_num = sum(
len(snapshots) for snapshots in nodes_stats.values())
logger.debug(
"Sending {snapshots} node stats snapshots about {nodes} nodes to the "
"AppScale Portal".format(snapshots=snapshots_num,
nodes=len(nodes_stats)))
request = helper.create_request(url=url,
method='POST',
body=urllib.urlencode(data))
response = helper.urlfetch(request)
if not response[JSONTags.SUCCESS]:
logger.error("Inaccessible resource: {}".format(url))
return
def send(self, nodes_stats):
deployment_id = helper.get_deployment_id()
# If the deployment is not registered, skip.
if not deployment_id:
return
# Send request to AppScale Portal.
portal_path = self._portal_method.format(deployment_id=deployment_id)
url = "{0}{1}".format(constants.PORTAL_URL, portal_path)
data = {
'deployment_id': deployment_id,
'nodes_stats': json.dumps({
node_ip: [stats_to_dict(snapshot) for snapshot in snapshots]
for node_ip, snapshots in nodes_stats.iteritems()
})
}
snapshots_num = sum(len(snapshots) for snapshots in nodes_stats.values())
logging.debug(
"Sending {snapshots} node stats snapshots about {nodes} nodes to the "
"AppScale Portal".format(snapshots=snapshots_num, nodes=len(nodes_stats))
)
request = helper.create_request(url=url, method='POST',
body=urllib.urlencode(data))
response = helper.urlfetch(request)
if not response[JSONTags.SUCCESS]:
logging.error("Inaccessible resource: {}".format(url))
return
def post(self):
""" POST method that sends a request for action to the
corresponding deployment components. """
logging.debug("Task request received: {0}, {1}".format(
str(self.request), str(self.request.body)))
if not self.request.body:
logging.info(
"Response from the AppScale Portal empty. No tasks to run.")
self.set_status(constants.HTTP_Codes.HTTP_OK)
return
try:
data = json.loads(self.request.body)
except (TypeError, ValueError) as error:
logging.exception(error)
logging.error("Unable to parse: {0}".format(self.request.body))
self.set_status(constants.HTTP_Codes.HTTP_BAD_REQUEST)
return
# Verify all necessary fields are present in request.body.
logging.debug("Verifying all necessary parameters are present.")
if not set(data.keys()).issuperset(set(constants.REQUIRED_KEYS)):
logging.error("Missing args in request: " + self.request.body)
self.set_status(constants.HTTP_Codes.HTTP_BAD_REQUEST)
return
# Gather information for sending the requests to start off the current
# task at hand.
nodes = helper.get_node_info()
if data[JSONTags.TYPE] not in constants.SUPPORTED_TASKS:
logging.error("Unsupported task type: '{0}'".format(
data[JSONTags.TYPE]))
self.set_status(constants.HTTP_Codes.HTTP_BAD_REQUEST)
return
tasks = [data[JSONTags.TYPE]]
logging.info("Tasks to execute: {0}".format(tasks))
for task in tasks:
# Initiate the task as pending.
TASK_STATUS_LOCK.acquire(True)
TASK_STATUS[data[JSONTags.TASK_ID]] = {
JSONTags.TYPE: task,
NodeInfoTags.NUM_NODES: len(nodes),
JSONTags.STATUS: TaskStatus.PENDING
}
TASK_STATUS_LOCK.release()
result_queue = Queue.Queue()
threads = []
for node in nodes:
# Create a br_service compatible JSON object.
json_data = helper.create_br_json_data(
node[NodeInfoTags.ROLE], task, data[JSONTags.BUCKET_NAME],
node[NodeInfoTags.INDEX], data[JSONTags.STORAGE])
request = helper.create_request(url=node[NodeInfoTags.HOST],
method='POST',
body=json_data)
# Start a thread for the request.
thread = threading.Thread(target=helper.send_remote_request,
name='{0}{1}'.format(
data[JSONTags.TYPE],
node[NodeInfoTags.HOST]),
args=(
request,
result_queue,
))
threads.append(thread)
thread.start()
# Wait for threads to finish.
for thread in threads:
thread.join()
# Harvest results.
results = [result_queue.get() for _ in xrange(len(nodes))]
logging.debug("Task: {0}. Results: {1}.".format(task, results))
# Backup source code.
app_success = False
if task == 'backup':
app_success = helper.\
backup_apps(data[JSONTags.STORAGE], data[JSONTags.BUCKET_NAME])
elif task == 'restore':
app_success = helper.\
restore_apps(data[JSONTags.STORAGE], data[JSONTags.BUCKET_NAME])
# Update TASK_STATUS.
successful_nodes = 0
for result in results:
if result[JSONTags.SUCCESS]:
successful_nodes += 1
TASK_STATUS_LOCK.acquire(True)
all_nodes = TASK_STATUS[data[JSONTags.TASK_ID]]\
[NodeInfoTags.NUM_NODES]
if successful_nodes < all_nodes or not app_success:
TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] = \
TaskStatus.FAILED
else:
TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] = \
TaskStatus.COMPLETE
logging.info("Task: {0}. Status: {1}.".format(
task, TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS]))
IOLoop.instance().add_callback(
callback=lambda: helper.report_status(
task, data[JSONTags.TASK_ID], TASK_STATUS[data[
JSONTags.TASK_ID]][JSONTags.STATUS]))
TASK_STATUS_LOCK.release()
self.set_status(constants.HTTP_Codes.HTTP_OK)