Example #1
0
    def test_get_deployment_id(self):
        # Test with a registered AppScale deployment.
        fake_acc = FakeAppControllerClient(True)
        flexmock(appscale_info).should_receive("get_appcontroller_client").and_return(fake_acc)
        flexmock(AppControllerClient).should_receive("deployment_id_exists").and_return(True)
        flexmock(AppControllerClient).should_receive("get_deployment_id").and_return("fake_id")
        self.assertEquals("fake_id", helper.get_deployment_id())

        # Test with an AppScale deployment that's not registered.
        fake_acc = FakeAppControllerClient(False)
        flexmock(appscale_info).should_receive("get_appcontroller_client").and_return(fake_acc)
        flexmock(AppControllerClient).should_receive("deployment_id_exists").and_return(False)
        self.assertIsNone(helper.get_deployment_id())
Example #2
0
def send_cluster_stats():
  """ Calls get_cluster_stats and sends the deployment monitoring stats to the
  AppScale Portal. """
  deployment_id = helper.get_deployment_id()
  # If the deployment is not registered, skip.
  if not deployment_id:
    return

  # Get all stats from this deployment.
  logging.debug("Getting all stats from every deployment node.")
  cluster_stats = helper.get_cluster_stats()

  # Send request to AppScale Portal.
  portal_path = hermes_constants.PORTAL_STATS_PATH.format(deployment_id)
  url = "{0}{1}".format(hermes_constants.PORTAL_URL, portal_path)
  data = {
    JSONTags.DEPLOYMENT_ID: deployment_id,
    JSONTags.TIMESTAMP: datetime.datetime.utcnow(),
    JSONTags.ALL_STATS: json.dumps(cluster_stats)
  }
  logging.debug("Sending all stats to the AppScale Portal. Data: \n{}".
    format(data))

  request = helper.create_request(url=url, method='POST',
    body=urllib.urlencode(data))
  response = helper.urlfetch(request)

  if not response[JSONTags.SUCCESS]:
    logging.error("Inaccessible resource: {}".format(url))
    return
Example #3
0
def send_all_stats():
  """ Calls get_all_stats and sends the deployment monitoring stats to the
  AppScale Portal. """
  deployment_id = helper.get_deployment_id()
  # If the deployment is not registered, skip.
  if not deployment_id:
    return

  # Get all stats from this deployment.
  logging.debug("Getting all stats from every deployment node.")
  all_stats = helper.get_all_stats()

  # Send request to AppScale Portal.
  portal_path = hermes_constants.PORTAL_STATS_PATH.format(deployment_id)
  url = "{0}{1}".format(hermes_constants.PORTAL_URL, portal_path)
  data = {
    JSONTags.DEPLOYMENT_ID: deployment_id,
    JSONTags.TIMESTAMP: datetime.datetime.utcnow(),
    JSONTags.ALL_STATS: json.dumps(all_stats)
  }
  logging.debug("Sending all stats to the AppScale Portal. Data: \n{}".
    format(data))

  request = helper.create_request(url=url, method='POST',
    body=urllib.urlencode(data))
  response = helper.urlfetch(request)

  if not response[JSONTags.SUCCESS]:
    logging.error("Inaccessible resource: {}".format(url))
    return
Example #4
0
def poll():
  """ Callback function that polls for new tasks based on a schedule. """
  logging.info("Polling for new task.")

  deployment_id = helper.get_deployment_id()
  # If the deployment is not registered, skip.
  if not deployment_id:
    return

  # Send request to AppScale Portal.
  url = "{0}{1}".format(hermes_constants.PORTAL_URL,
      hermes_constants.PORTAL_POLL_PATH)
  data = json.dumps({ JSONTags.DEPLOYMENT_ID: deployment_id })
  request = helper.create_request(url=url, method='POST', body=data)
  response = helper.urlfetch(request)
  try:
    data = json.loads(response.body)
  except (TypeError, ValueError) as error:
    logging.error("Cannot parse response from url '{0}'. Error: {1}".
      format(url, str(error)))
    return

  # Verify all necessary fields are present in the request.
  if not set(data.keys()).issuperset(set(hermes_constants.REQUIRED_KEYS)) or \
      None in data.values():
    logging.error("Missing args in response: {0}".format(response))
    return

  logging.debug("Task to run: {0}".format(data))
  logging.info("Redirecting task request to TaskHandler.")
  url = "{0}{1}".format(hermes_constants.HERMES_URL, TaskHandler.PATH)
  request = helper.create_request(url, method='POST', body=data)

  # The poller can move forward without waiting for a response here.
  helper.urlfetch_async(request)
Example #5
0
  def test_get_deployment_id(self):
    # Test with a registered AppScale deployment.
    flexmock(helper).should_receive('read_file_contents').with_args(
      '/etc/appscale/head_node_ip').and_return('foo')
    flexmock(helper).should_receive('read_file_contents').with_args(
      '/etc/appscale/secret.key').and_return('bar')
    flexmock(AppControllerClient).should_receive('deployment_id_exists').\
      and_return(True)
    flexmock(AppControllerClient).should_receive('get_deployment_id').\
      and_return('fake_id')
    self.assertEquals('fake_id', helper.get_deployment_id())

    # Test with an AppScale deployment that's not registered.
    flexmock(AppControllerClient).should_receive('deployment_id_exists').\
      and_return(False)
    self.assertIsNone(helper.get_deployment_id())
Example #6
0
def poll():
  """ Callback function that polls for new tasks based on a schedule. """
  deployment_id = helper.get_deployment_id()
  # If the deployment is not registered, skip.
  if not deployment_id:
    return

  # If we can't reach the backup and recovery services, skip.
  nodes = helper.get_node_info()
  http_client = tornado.httpclient.HTTPClient()
  for node in nodes:
    br_host = node[helper.NodeInfoTags.HOST]
    request = tornado.httpclient.HTTPRequest(br_host)
    try:
      response = http_client.fetch(request)
      if json.loads(response.body)['status'] != 'up':
        logging.warn('Backup and Recovery service at {} is not up.'
          .format(br_host))
        return
    except (socket.error, ValueError):
      logging.exception('Backup and Recovery service at {} is not up.'
        .format(br_host))
      return

  logging.info("Polling for new task.")

  # Send request to AppScale Portal.
  url = "{0}{1}".format(hermes_constants.PORTAL_URL,
      hermes_constants.PORTAL_POLL_PATH)
  data = urllib.urlencode({JSONTags.DEPLOYMENT_ID: deployment_id})
  request = helper.create_request(url=url, method='POST', body=data)
  response = helper.urlfetch(request)

  if not response[JSONTags.SUCCESS]:
    logging.error("Inaccessible resource: {}".format(url))
    return

  try:
    data = json.loads(response[JSONTags.BODY])
  except (TypeError, ValueError) as error:
    logging.error("Cannot parse response from url '{0}'. Error: {1}".
      format(url, str(error)))
    return

  if data == {}:  # If there's no task to perform.
    return

  # Verify all necessary fields are present in the request.
  if not set(data.keys()).issuperset(set(hermes_constants.REQUIRED_KEYS)):
    logging.error("Missing args in response: {0}".format(response))
    return

  logging.debug("Task to run: {0}".format(data))
  logging.info("Redirecting task request to TaskHandler.")
  url = "{0}{1}".format(hermes_constants.HERMES_URL, TaskHandler.PATH)
  request = helper.create_request(url, method='POST', body=json.dumps(data))

  # The poller can move forward without waiting for a response here.
  helper.urlfetch_async(request)
Example #7
0
def poll():
  """ Callback function that polls for new tasks based on a schedule. """
  deployment_id = helper.get_deployment_id()
  # If the deployment is not registered, skip.
  if not deployment_id:
    return

  # If we can't reach the backup and recovery services, skip.
  nodes = helper.get_node_info()
  http_client = tornado.httpclient.HTTPClient()
  for node in nodes:
    br_host = node[helper.NodeInfoTags.HOST]
    request = tornado.httpclient.HTTPRequest(br_host)
    try:
      response = http_client.fetch(request)
      if json.loads(response.body)['status'] != 'up':
        logging.warn('Backup and Recovery service at {} is not up.'
          .format(br_host))
        return
    except (socket.error, ValueError):
      logging.exception('Backup and Recovery service at {} is not up.'
        .format(br_host))
      return

  logging.info("Polling for new task.")

  # Send request to AppScale Portal.
  url = "{0}{1}".format(hermes_constants.PORTAL_URL,
      hermes_constants.PORTAL_POLL_PATH)
  data = urllib.urlencode({JSONTags.DEPLOYMENT_ID: deployment_id})
  request = helper.create_request(url=url, method='POST', body=data)
  response = helper.urlfetch(request)

  if not response[JSONTags.SUCCESS]:
    logging.error("Inaccessible resource: {}".format(url))
    return

  try:
    data = json.loads(response[JSONTags.BODY])
  except (TypeError, ValueError) as error:
    logging.error("Cannot parse response from url '{0}'. Error: {1}".
      format(url, str(error)))
    return

  if data == {}:  # If there's no task to perform.
    return

  # Verify all necessary fields are present in the request.
  if not set(data.keys()).issuperset(set(hermes_constants.REQUIRED_KEYS)):
    logging.error("Missing args in response: {0}".format(response))
    return

  logging.debug("Task to run: {0}".format(data))
  logging.info("Redirecting task request to TaskHandler.")
  url = "{0}{1}".format(hermes_constants.HERMES_URL, TaskHandler.PATH)
  request = helper.create_request(url, method='POST', body=json.dumps(data))

  # The poller can move forward without waiting for a response here.
  helper.urlfetch_async(request)
Example #8
0
    def test_get_deployment_id(self):
        # Test with a registered AppScale deployment.
        fake_acc = FakeAppControllerClient(True)
        flexmock(appscale_info).should_receive('get_appcontroller_client').\
          and_return(fake_acc)
        flexmock(AppControllerClient).should_receive('deployment_id_exists').\
          and_return(True)
        flexmock(AppControllerClient).should_receive('get_deployment_id').\
          and_return('fake_id')
        self.assertEquals('fake_id', helper.get_deployment_id())

        # Test with an AppScale deployment that's not registered.
        fake_acc = FakeAppControllerClient(False)
        flexmock(appscale_info).should_receive('get_appcontroller_client').\
          and_return(fake_acc)
        flexmock(AppControllerClient).should_receive('deployment_id_exists').\
          and_return(False)
        self.assertIsNone(helper.get_deployment_id())
Example #9
0
def deploy_sensor_app():
    """ Uploads the sensor app for registered deployments. """

    deployment_id = helper.get_deployment_id()
    #If deployment is not registered, then do nothing.
    if not deployment_id:
        return

    uaserver = SOAPpy.SOAPProxy('https://{0}:{1}'.format(
        appscale_info.get_db_master_ip(), hermes_constants.UA_SERVER_PORT))

    # If the appscalesensor app is already running, then do nothing.
    is_app_enabled = uaserver.is_app_enabled(hermes_constants.APPSCALE_SENSOR,
                                             appscale_info.get_secret())
    if is_app_enabled == "true":
        return

    pwd = appscale_utils.encrypt_password(
        hermes_constants.USER_EMAIL,
        appscale_utils.random_password_generator())
    if create_appscale_user(pwd, uaserver) and create_xmpp_user(pwd, uaserver):
        logging.debug("Created new user and now tarring app to be deployed.")
        file_path = os.path.join(os.path.dirname(__file__), '../Apps/sensor')
        app_dir_location = os.path.join(hermes_constants.APP_DIR_LOCATION,
                                        hermes_constants.APPSCALE_SENSOR)
        archive = tarfile.open(app_dir_location, "w|gz")
        archive.add(file_path, arcname=hermes_constants.APPSCALE_SENSOR)
        archive.close()

        try:
            logging.info(
                "Deploying the sensor app for registered deployments.")
            acc = appscale_info.get_appcontroller_client()
            acc.upload_app(app_dir_location, hermes_constants.FILE_SUFFIX,
                           hermes_constants.USER_EMAIL)
        except AppControllerException:
            logging.exception("AppControllerException while trying to deploy "
                              "appscalesensor app.")
    else:
        logging.error("Error while creating or accessing the user to deploy "
                      "appscalesensor app.")
Example #10
0
def deploy_sensor_app():
  """ Uploads the sensor app for registered deployments. """

  deployment_id = helper.get_deployment_id()
  #If deployment is not registered, then do nothing.
  if not deployment_id:
    return

  uaserver = SOAPpy.SOAPProxy('https://{0}:{1}'.format(
    appscale_info.get_db_master_ip(), hermes_constants.UA_SERVER_PORT))

  # If the appscalesensor app is already running, then do nothing.
  is_app_enabled = uaserver.is_app_enabled(hermes_constants.APPSCALE_SENSOR,
    appscale_info.get_secret())
  if is_app_enabled == "true":
    return

  pwd = appscale_utils.encrypt_password(hermes_constants.USER_EMAIL,
    appscale_utils.random_password_generator())
  if create_appscale_user(pwd, uaserver) and create_xmpp_user(pwd, uaserver):
    logging.debug("Created new user and now tarring app to be deployed.")
    file_path = os.path.join(os.path.dirname(__file__), '../Apps/sensor')
    app_dir_location = os.path.join(hermes_constants.APP_DIR_LOCATION,
      hermes_constants.APPSCALE_SENSOR)
    archive = tarfile.open(app_dir_location, "w|gz")
    archive.add(file_path, arcname= hermes_constants.APPSCALE_SENSOR)
    archive.close()

    try:
      logging.info("Deploying the sensor app for registered deployments.")
      acc = appscale_info.get_appcontroller_client()
      acc.upload_app(app_dir_location, hermes_constants.FILE_SUFFIX,
        hermes_constants.USER_EMAIL)
    except AppControllerException:
      logging.exception("AppControllerException while trying to deploy "
        "appscalesensor app.")
  else:
    logging.error("Error while creating or accessing the user to deploy "
      "appscalesensor app.")
Example #11
0
def poll():
    """ Callback function that polls for new tasks based on a schedule. """
    logging.info("Polling for new task.")

    deployment_id = helper.get_deployment_id()
    # If the deployment is not registered, skip.
    if not deployment_id:
        return

    # Send request to AppScale Portal.
    url = "{0}{1}".format(hermes_constants.PORTAL_URL,
                          hermes_constants.PORTAL_POLL_PATH)
    data = json.dumps({JSONTags.DEPLOYMENT_ID: deployment_id})
    request = helper.create_request(url=url, method='POST', body=data)
    response = helper.urlfetch(request)
    try:
        data = json.loads(response.body)
    except (TypeError, ValueError) as error:
        logging.error(
            "Cannot parse response from url '{0}'. Error: {1}".format(
                url, str(error)))
        return

    # Verify all necessary fields are present in the request.
    if not set(data.keys()).issuperset(set(hermes_constants.REQUIRED_KEYS)) or \
        None in data.values():
        logging.error("Missing args in response: {0}".format(response))
        return

    logging.debug("Task to run: {0}".format(data))
    logging.info("Redirecting task request to TaskHandler.")
    url = "{0}{1}".format(hermes_constants.HERMES_URL, TaskHandler.PATH)
    request = helper.create_request(url, method='POST', body=data)

    # The poller can move forward without waiting for a response here.
    helper.urlfetch_async(request)