Exemplo n.º 1
0
    def test_stop(self):
        YAML_INFO = """--- 
:keyname: appscale
:replication: "1"
:table: cassandra
"""
        flexmock(file_io).should_receive('read').and_return(YAML_INFO)
        self.assertEqual('cassandra', appscale_info.get_db_info()[':table'])
        self.assertEqual('1', appscale_info.get_db_info()[':replication'])
        self.assertEqual('appscale', appscale_info.get_db_info()[':keyname'])
        self.assertEqual(True, isinstance(appscale_info.get_db_info(), dict))
Exemplo n.º 2
0
  def test_stop(self):
    YAML_INFO="""--- 
:keyname: appscale
:replication: "1"
:table: cassandra
"""
    flexmock(file_io).should_receive('read').and_return(YAML_INFO)
    self.assertEqual('cassandra', appscale_info.get_db_info()[':table'])
    self.assertEqual( '1', appscale_info.get_db_info()[':replication'])
    self.assertEqual( 'appscale', appscale_info.get_db_info()[':keyname'])
    self.assertEqual(True, isinstance(appscale_info.get_db_info(), dict))
Exemplo n.º 3
0
def main():
  """ This main function allows you to run the groomer manually. """
  zk_connection_locations = appscale_info.get_zk_locations_string()
  zookeeper = zk.ZKTransaction(host=zk_connection_locations)
  db_info = appscale_info.get_db_info()
  table = db_info[':table']
  master = appscale_info.get_db_master_ip()
  datastore_path = "{0}:8888".format(master)
  ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)

  logging.debug("Trying to get groomer lock.")
  if ds_groomer.get_groomer_lock():
    logging.info("Got the groomer lock.")
    try:
      ds_groomer.run_groomer()
    except Exception as exception:
      logging.exception('Encountered exception {} while running the groomer.'
        .format(str(exception)))
    try:
      ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
    except zk.ZKTransactionException, zk_exception:
      logging.error("Unable to release zk lock {0}.".\
        format(str(zk_exception)))
    except zk.ZKInternalException, zk_exception:
      logging.error("Unable to release zk lock {0}.".\
        format(str(zk_exception)))
Exemplo n.º 4
0
def main():
  """ This main function allows you to run the backup manually. """

  parser = init_parser()
  args = parser.parse_args()

  # Set up logging.
  level = logging.INFO
  if args.debug:
    level = logging.DEBUG
  logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
    '%(lineno)s %(message)s ', level=level)
  logging.info("Logging started")

  message = "Backing up "
  if args.source_code:
    message += "source and "
  message += "data for: {0}".format(args.app_id)
  logging.info(message)

  zk_connection_locations = appscale_info.get_zk_locations_string()
  zookeeper = zk.ZKTransaction(host=zk_connection_locations)
  db_info = appscale_info.get_db_info()
  table = db_info[':table']

  skip_list = args.skip
  if not skip_list:
    skip_list = []
  logging.info("Will skip the following kinds: {0}".format(sorted(skip_list)))
  ds_backup = DatastoreBackup(args.app_id, zookeeper, table,
    source_code=args.source_code, skip_list=sorted(skip_list))
  try:
    ds_backup.run()
  finally:
    zookeeper.close()
Exemplo n.º 5
0
def main():
  """ This main function allows you to run the groomer manually. """
  zk_connection_locations = appscale_info.get_zk_locations_string()
  zookeeper = zk.ZKTransaction(host=zk_connection_locations, start_gc=False)
  db_info = appscale_info.get_db_info()
  table = db_info[':table']
  master = appscale_info.get_db_master_ip()
  datastore_path = "{0}:8888".format(master)
  ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)

  logging.debug("Trying to get groomer lock.")
  if ds_groomer.get_groomer_lock():
    logging.info("Got the groomer lock.")
    try:
      ds_groomer.run_groomer()
    except Exception as exception:
      logging.exception('Encountered exception {} while running the groomer.'
        .format(str(exception)))
    try:
      ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
    except zk.ZKTransactionException, zk_exception:
      logging.error("Unable to release zk lock {0}.".\
        format(str(zk_exception)))
    except zk.ZKInternalException, zk_exception:
      logging.error("Unable to release zk lock {0}.".\
        format(str(zk_exception)))
Exemplo n.º 6
0
def main():
    """ This main function allows you to run the restore manually. """

    # Parse CLI arguments.
    parser = init_parser()
    args = parser.parse_args()

    # Set up logging.
    level = logging.INFO
    if args.debug:
        level = logging.DEBUG
    logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
      '%(lineno)s %(message)s ', level=level)
    logging.info("Logging started")

    logging.info(args)

    # Verify app is deployed.
    if not app_is_deployed(args.app_id):
        return

    # Verify backup dir exists.
    if not backup_dir_exists(args.backup_dir):
        return

    if args.clear_datastore:
        message = "Deleting \"{0}\" data...".\
          format(args.app_id, args.backup_dir)
        logging.info(message)
        try:
            tables_to_clear = {
                APP_ENTITY_TABLE: APP_ENTITY_SCHEMA,
                ASC_PROPERTY_TABLE: PROPERTY_SCHEMA,
                DSC_PROPERTY_TABLE: PROPERTY_SCHEMA,
                COMPOSITE_TABLE: COMPOSITE_SCHEMA,
                APP_KIND_TABLE: APP_KIND_SCHEMA,
                TRANSACTIONS_TABLE: TRANSACTIONS_SCHEMA
            }
            for table, schema in tables_to_clear.items():
                fetch_and_delete_entities('cassandra', table, schema,
                                          args.app_id, False)
        except Exception as exception:
            logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \
              "Exiting...".format(args.app_id, exception.message))
            return

    # Initialize connection to Zookeeper and database related variables.
    zk_connection_locations = appscale_info.get_zk_locations_string()
    zookeeper = zk.ZKTransaction(host=zk_connection_locations)
    db_info = appscale_info.get_db_info()
    table = db_info[':table']

    # Start restore process.
    ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir,
                                  zookeeper, table)
    try:
        ds_restore.run()
    finally:
        zookeeper.close()
Exemplo n.º 7
0
def get_datastore():
  """ Returns a reference to the batch datastore interface. Validates where
  the <database>_interface.py is and adds that path to the system path.
  """
  db_info = appscale_info.get_db_info()
  db_type = db_info[':table']
  datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(db_type)
  return datastore_batch
Exemplo n.º 8
0
def get_datastore():
  """ Returns a reference to the batch datastore interface. Validates where
  the <database>_interface.py is and adds that path to the system path.
  """
  db_info = appscale_info.get_db_info()
  db_type = db_info[':table']
  datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(db_type)
  return datastore_batch
Exemplo n.º 9
0
def main():
  """ This main function allows you to run the restore manually. """

  # Parse CLI arguments.
  parser = init_parser()
  args = parser.parse_args()

  # Set up logging.
  level = logging.INFO
  if args.debug:
    level = logging.DEBUG
  logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
    '%(lineno)s %(message)s ', level=level)
  logging.info("Logging started")

  logging.info(args)

  # Verify app is deployed.
  if not app_is_deployed(args.app_id):
    return

  # Verify backup dir exists.
  if not backup_dir_exists(args.backup_dir):
    return

  if args.clear_datastore:
    message = "Deleting \"{0}\" data...".\
      format(args.app_id, args.backup_dir)
    logging.info(message)
    try:
      tables_to_clear = {
        APP_ENTITY_TABLE: APP_ENTITY_SCHEMA,
        ASC_PROPERTY_TABLE: PROPERTY_SCHEMA,
        DSC_PROPERTY_TABLE: PROPERTY_SCHEMA,
        COMPOSITE_TABLE: COMPOSITE_SCHEMA,
        APP_KIND_TABLE: APP_KIND_SCHEMA,
        TRANSACTIONS_TABLE: TRANSACTIONS_SCHEMA
      }
      for table, schema in tables_to_clear.items():
        fetch_and_delete_entities('cassandra', table, schema, args.app_id, False)
    except Exception as exception:
      logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \
        "Exiting...".format(args.app_id, exception.message))
      return

  # Initialize connection to Zookeeper and database related variables.
  zk_connection_locations = appscale_info.get_zk_locations_string()
  zookeeper = zk.ZKTransaction(host=zk_connection_locations)
  db_info = appscale_info.get_db_info()
  table = db_info[':table']

  # Start restore process.
  ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir,
    zookeeper, table)
  try:
    ds_restore.run()
  finally:
    zookeeper.close()
Exemplo n.º 10
0
def main():
  """ This main function allows you to run the groomer manually. """
  zookeeper = zk.ZKTransaction(host="localhost:2181")
  datastore_path = "localhost:8888"
  db_info = appscale_info.get_db_info()
  table = db_info[':table']
  ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
  try:
    ds_groomer.run_groomer()
  finally:
    zookeeper.close()
Exemplo n.º 11
0
def main():
    """ This main function allows you to run the groomer manually. """
    zk_connection_locations = appscale_info.get_zk_locations_string()
    zookeeper = zk.ZKTransaction(host=zk_connection_locations)
    datastore_path = "localhost:8888"
    db_info = appscale_info.get_db_info()
    table = db_info[":table"]
    ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
    try:
        ds_groomer.run_groomer()
    finally:
        zookeeper.close()
Exemplo n.º 12
0
def main():
  """ This main function allows you to run the groomer manually. """
  zk_connection_locations = appscale_info.get_zk_locations_string()
  zookeeper = zk.ZKTransaction(host=zk_connection_locations)
  db_info = appscale_info.get_db_info()
  table = db_info[':table']
  master = appscale_info.get_db_master_ip()
  datastore_path = "{0}:8888".format(master)
  ds_groomer = groomer.DatastoreGroomer(zookeeper, table, datastore_path)
  try:
    ds_groomer.remove_old_logs(None)
  finally:
    zookeeper.close()
Exemplo n.º 13
0
def main():
    """ This main function allows you to run the groomer manually. """
    zk_connection_locations = appscale_info.get_zk_locations_string()
    zookeeper = zk.ZKTransaction(host=zk_connection_locations)
    db_info = appscale_info.get_db_info()
    table = db_info[':table']
    master = appscale_info.get_db_master_ip()
    datastore_path = "{0}:8888".format(master)
    ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
    try:
        ds_groomer.run_groomer()
    finally:
        zookeeper.close()
Exemplo n.º 14
0
def main():
    """ Starts a web service for handing datastore requests. """

    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
    global logger
    logger = logging.getLogger(__name__)

    global datastore_access
    zookeeper_locations = appscale_info.get_zk_locations_string()

    db_info = appscale_info.get_db_info()
    db_type = db_info[':table']
    port = dbconstants.DEFAULT_SSL_PORT
    is_encrypted = True
    verbose = False

    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(
            argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"])
    except getopt.GetoptError:
        usage()
        sys.exit(1)

    for opt, arg in opts:
        if opt in ("-t", "--type"):
            db_type = arg
            print "Datastore type: ", db_type
        elif opt in ("-p", "--port"):
            port = int(arg)
        elif opt in ("-n", "--no_encryption"):
            is_encrypted = False
        elif opt in ("-v", "--verbose"):
            verbose = True

    if verbose:
        logger.setLevel(logging.DEBUG)

    if db_type not in dbconstants.VALID_DATASTORES:
        print "This datastore is not supported for this version of the AppScale\
          datastore API:" + db_type
        sys.exit(1)

    datastore_batch = DatastoreFactory.getDatastore(
        db_type, log_level=logger.getEffectiveLevel())
    zookeeper = zktransaction.ZKTransaction(
        host=zookeeper_locations,
        start_gc=True,
        db_access=datastore_batch,
        log_level=logger.getEffectiveLevel())

    datastore_access = DatastoreDistributed(
        datastore_batch,
        zookeeper=zookeeper,
        log_level=logger.getEffectiveLevel())
    if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted:
        port = dbconstants.DEFAULT_PORT

    server = tornado.httpserver.HTTPServer(pb_application)
    server.listen(port)

    while 1:
        try:
            # Start Server #
            tornado.ioloop.IOLoop.instance().start()
        except SSL.SSLError:
            # This happens when connections timeout, there is a just a bad
            # SSL connection such as it does not use SSL when expected.
            pass
        except KeyboardInterrupt:
            print "Server interrupted by user, terminating..."
            zookeeper.close()
            sys.exit(1)
Exemplo n.º 15
0
def main():
    """ Starts a web service for handing datastore requests. """

    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
    global logger
    logger = logging.getLogger(__name__)

    global datastore_access
    zookeeper_locations = appscale_info.get_zk_locations_string()

    db_info = appscale_info.get_db_info()
    db_type = db_info[":table"]
    port = dbconstants.DEFAULT_SSL_PORT
    is_encrypted = True
    verbose = False

    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"])
    except getopt.GetoptError:
        usage()
        sys.exit(1)

    for opt, arg in opts:
        if opt in ("-t", "--type"):
            db_type = arg
            print "Datastore type: ", db_type
        elif opt in ("-p", "--port"):
            port = int(arg)
        elif opt in ("-n", "--no_encryption"):
            is_encrypted = False
        elif opt in ("-v", "--verbose"):
            verbose = True

    if verbose:
        logger.setLevel(logging.DEBUG)

    if db_type not in dbconstants.VALID_DATASTORES:
        print "This datastore is not supported for this version of the AppScale\
          datastore API:" + db_type
        sys.exit(1)

    datastore_batch = DatastoreFactory.getDatastore(db_type, log_level=logger.getEffectiveLevel())
    zookeeper = zktransaction.ZKTransaction(
        host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel()
    )

    datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel())
    if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted:
        port = dbconstants.DEFAULT_PORT

    server = tornado.httpserver.HTTPServer(pb_application)
    server.listen(port)

    while 1:
        try:
            # Start Server #
            tornado.ioloop.IOLoop.instance().start()
        except SSL.SSLError:
            # This happens when connections timeout, there is a just a bad
            # SSL connection such as it does not use SSL when expected.
            pass
        except KeyboardInterrupt:
            print "Server interrupted by user, terminating..."
            zookeeper.close()
            sys.exit(1)
Exemplo n.º 16
0
  if not backup_dir_exists(args.backup_dir):
    return

  if args.clear_datastore:
    message = "Deleting \"{0}\" data...".\
      format(args.app_id, args.backup_dir)
    logging.info(message)
    try:
      delete_all_records.main('cassandra', args.app_id, True)
    except Exception, exception:
      logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \
        "Exiting...".format(args.app_id, exception.message))
      return

  # Initialize connection to Zookeeper and database related variables.
  zk_connection_locations = appscale_info.get_zk_locations_string()
  zookeeper = zk.ZKTransaction(host=zk_connection_locations)
  db_info = appscale_info.get_db_info()
  table = db_info[':table']

  # Start restore process.
  ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir,
    zookeeper, table)
  try:
    ds_restore.run()
  finally:
    zookeeper.close()

if __name__ == "__main__":
  main()