def poll(): """ Callback function that polls for new tasks based on a schedule. """ deployment_id = helper.get_deployment_id() # If the deployment is not registered, skip. if not deployment_id: return # If we can't reach the backup and recovery services, skip. nodes = helper.get_node_info() http_client = tornado.httpclient.HTTPClient() for node in nodes: br_host = node[helper.NodeInfoTags.HOST] request = tornado.httpclient.HTTPRequest(br_host) try: response = http_client.fetch(request) if json.loads(response.body)['status'] != 'up': logging.warn('Backup and Recovery service at {} is not up.' .format(br_host)) return except (socket.error, ValueError): logging.exception('Backup and Recovery service at {} is not up.' .format(br_host)) return logging.info("Polling for new task.") # Send request to AppScale Portal. url = "{0}{1}".format(hermes_constants.PORTAL_URL, hermes_constants.PORTAL_POLL_PATH) data = urllib.urlencode({JSONTags.DEPLOYMENT_ID: deployment_id}) request = helper.create_request(url=url, method='POST', body=data) response = helper.urlfetch(request) if not response[JSONTags.SUCCESS]: logging.error("Inaccessible resource: {}".format(url)) return try: data = json.loads(response[JSONTags.BODY]) except (TypeError, ValueError) as error: logging.error("Cannot parse response from url '{0}'. Error: {1}". format(url, str(error))) return if data == {}: # If there's no task to perform. return # Verify all necessary fields are present in the request. if not set(data.keys()).issuperset(set(hermes_constants.REQUIRED_KEYS)): logging.error("Missing args in response: {0}".format(response)) return logging.debug("Task to run: {0}".format(data)) logging.info("Redirecting task request to TaskHandler.") url = "{0}{1}".format(hermes_constants.HERMES_URL, TaskHandler.PATH) request = helper.create_request(url, method='POST', body=json.dumps(data)) # The poller can move forward without waiting for a response here. helper.urlfetch_async(request)
def test_get_node_info(self): flexmock(appscale_info).should_receive('get_db_master_ip').and_return( 'foo') flexmock(appscale_info).should_receive('get_db_slave_ips').and_return( ['bar']) flexmock(appscale_info).should_receive('get_zk_node_ips').and_return( ['baz']) flexmock(helper).should_receive('get_br_service_url').and_return( 'http://some.url').at_least().times(2) self.assertEquals(fake_node_info, helper.get_node_info())
def test_get_node_info(self): flexmock(appscale_info).should_receive("get_db_master_ip").and_return("foo") flexmock(appscale_info).should_receive("get_db_slave_ips").and_return(["bar"]) flexmock(appscale_info).should_receive("get_zk_node_ips").and_return(["baz"]) flexmock(helper).should_receive("get_br_service_url").and_return("http://some.url").at_least().times(2) self.assertEquals(fake_node_info, helper.get_node_info())
def post(self): """ POST method that sends a request for action to the corresponding deployment components. """ logging.info("Task request received: {0}, {1}".format(str(self.request), str(self.request.body))) try: data = json.loads(self.request.body) except (TypeError, ValueError) as error: logging.exception(error) logging.error("Unable to parse: {0}".format(self.request.body)) self.set_status(hermes_constants.HTTP_Codes.HTTP_BAD_REQUEST) return # Verify all necessary fields are present in request.body. logging.info("Verifying all necessary parameters are present.") if not set(data.keys()).issuperset(set(hermes_constants.REQUIRED_KEYS)): logging.error("Missing args in request: " + self.request.body) self.set_status(hermes_constants.HTTP_Codes.HTTP_BAD_REQUEST) return # Gather information for sending the requests to start off the current # task at hand. nodes = helper.get_node_info() if data[JSONTags.TYPE] == 'backup' or data[JSONTags.TYPE] == 'restore': tasks = [data[JSONTags.TYPE]] else: logging.error("Unsupported task type: '{0}'".format(data[JSONTags.TYPE])) self.set_status(hermes_constants.HTTP_Codes.HTTP_BAD_REQUEST) return logging.debug("Tasks to execute: {0}".format(tasks)) for task in tasks: # Initiate the task as pending. TASK_STATUS_LOCK.acquire(True) TASK_STATUS[data[JSONTags.TASK_ID]] = { JSONTags.TYPE: task, NodeInfoTags.NUM_NODES: len(nodes), JSONTags.STATUS: TaskStatus.PENDING } TASK_STATUS_LOCK.release() result_queue = Queue.Queue() threads = [] for node in nodes: # Create a br_service compatible JSON object. json_data = helper.create_br_json_data( node[NodeInfoTags.ROLE], task, data[JSONTags.BUCKET_NAME], node[NodeInfoTags.INDEX], data[JSONTags.STORAGE]) request = helper.create_request(url=node[NodeInfoTags.HOST], method='POST', body=json_data) # Start a thread for the request. thread = threading.Thread( target=helper.send_remote_request, name='{0}{1}'. format(data[JSONTags.TYPE], node[NodeInfoTags.HOST]), args=(request, result_queue,)) threads.append(thread) thread.start() # Wait for threads to finish. for thread in threads: thread.join() # Harvest results. results = [result_queue.get() for _ in xrange(len(nodes))] logging.debug("Task: {0}. Results: {1}.".format(task, results)) # Backup source code. app_success = False if task == 'backup': app_success = helper.\ backup_apps(data[JSONTags.STORAGE], data[JSONTags.BUCKET_NAME]) elif task == 'restore': app_success = helper.\ restore_apps(data[JSONTags.STORAGE], data[JSONTags.BUCKET_NAME]) # Update TASK_STATUS. successful_nodes = 0 for result in results: if result[JSONTags.SUCCESS]: successful_nodes += 1 TASK_STATUS_LOCK.acquire(True) all_nodes = TASK_STATUS[data[JSONTags.TASK_ID]]\ [NodeInfoTags.NUM_NODES] if successful_nodes < all_nodes: TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] = \ TaskStatus.FAILED else: TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] = \ TaskStatus.COMPLETE logging.info("Task: {0}. Status: {1}.".format(task, TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS])) IOLoop.instance().add_callback(callback=lambda: helper.report_status(task, data[JSONTags.TASK_ID], TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] )) TASK_STATUS_LOCK.release() self.set_status(hermes_constants.HTTP_Codes.HTTP_OK)
def post(self): """ POST method that sends a request for action to the corresponding deployment components. """ logging.info("Task request received: {0}, {1}".format( str(self.request), str(self.request.body))) try: data = json.loads(self.request.body) except (TypeError, ValueError) as error: logging.exception(error) logging.error("Unable to parse: {0}".format(self.request.body)) self.set_status(hermes_constants.HTTP_Codes.HTTP_BAD_REQUEST) return # Verify all necessary fields are present in request.body. logging.info("Verifying all necessary parameters are present.") if not set(data.keys()).issuperset(set( hermes_constants.REQUIRED_KEYS)): logging.error("Missing args in request: " + self.request.body) self.set_status(hermes_constants.HTTP_Codes.HTTP_BAD_REQUEST) return # Gather information for sending the requests to start off the current # task at hand. nodes = helper.get_node_info() # Ensure that we bring down affected nodes before any action while doing a # restore. if data[JSONTags.TYPE] == 'backup': tasks = [data[JSONTags.TYPE]] elif data[JSONTags.TYPE] == 'restore': tasks = ['restore'] else: logging.error("Unsupported task type: '{0}'".format( data[JSONTags.TYPE])) self.set_status(hermes_constants.HTTP_Codes.HTTP_BAD_REQUEST) return logging.info("Tasks to execute: {0}".format(tasks)) for task in tasks: # Initiate the task as pending. TASK_STATUS_LOCK.acquire(True) TASK_STATUS[data[JSONTags.TASK_ID]] = { JSONTags.TYPE: task, NodeInfoTags.NUM_NODES: len(nodes), JSONTags.STATUS: TaskStatus.PENDING } TASK_STATUS_LOCK.release() result_queue = Queue.Queue() threads = [] for node in nodes: # Create a br_service compatible JSON object. json_data = helper.create_br_json_data( node[NodeInfoTags.ROLE], task, data[JSONTags.BUCKET_NAME], node[NodeInfoTags.INDEX], data[JSONTags.STORAGE]) request = helper.create_request(url=node[NodeInfoTags.HOST], method='POST', body=json_data) # Start a thread for the request. thread = threading.Thread(target=helper.send_remote_request, name='{0}{1}'.format( data[JSONTags.TYPE], node[NodeInfoTags.HOST]), args=( request, result_queue, )) threads.append(thread) thread.start() # Wait for threads to finish. for thread in threads: thread.join() # Harvest results. results = [result_queue.get() for _ in xrange(len(nodes))] logging.warn("Task: {0}. Results: {1}.".format(task, results)) # Update TASK_STATUS. successful_nodes = 0 for result in results: if result[JSONTags.SUCCESS]: successful_nodes += 1 TASK_STATUS_LOCK.acquire(True) all_nodes = TASK_STATUS[data[JSONTags.TASK_ID]]\ [NodeInfoTags.NUM_NODES] if successful_nodes < all_nodes: TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] = \ TaskStatus.FAILED else: TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS] = \ TaskStatus.COMPLETE logging.info("Task: {0}. Status: {1}.".format( task, TASK_STATUS[data[JSONTags.TASK_ID]][JSONTags.STATUS])) IOLoop.instance().add_callback( callback=lambda: helper.report_status( task, data[JSONTags.TASK_ID], TASK_STATUS[data[ JSONTags.TASK_ID]][JSONTags.STATUS])) TASK_STATUS_LOCK.release() self.set_status(hermes_constants.HTTP_Codes.HTTP_OK)