def test_stop(self): YAML_INFO = """--- :keyname: appscale :replication: "1" :table: cassandra """ flexmock(file_io).should_receive('read').and_return(YAML_INFO) self.assertEqual('cassandra', appscale_info.get_db_info()[':table']) self.assertEqual('1', appscale_info.get_db_info()[':replication']) self.assertEqual('appscale', appscale_info.get_db_info()[':keyname']) self.assertEqual(True, isinstance(appscale_info.get_db_info(), dict))
def test_stop(self): YAML_INFO="""--- :keyname: appscale :replication: "1" :table: cassandra """ flexmock(file_io).should_receive('read').and_return(YAML_INFO) self.assertEqual('cassandra', appscale_info.get_db_info()[':table']) self.assertEqual( '1', appscale_info.get_db_info()[':replication']) self.assertEqual( 'appscale', appscale_info.get_db_info()[':keyname']) self.assertEqual(True, isinstance(appscale_info.get_db_info(), dict))
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) logging.debug("Trying to get groomer lock.") if ds_groomer.get_groomer_lock(): logging.info("Got the groomer lock.") try: ds_groomer.run_groomer() except Exception as exception: logging.exception('Encountered exception {} while running the groomer.' .format(str(exception))) try: ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH) except zk.ZKTransactionException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception))) except zk.ZKInternalException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception)))
def main(): """ This main function allows you to run the backup manually. """ parser = init_parser() args = parser.parse_args() # Set up logging. level = logging.INFO if args.debug: level = logging.DEBUG logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \ '%(lineno)s %(message)s ', level=level) logging.info("Logging started") message = "Backing up " if args.source_code: message += "source and " message += "data for: {0}".format(args.app_id) logging.info(message) zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] skip_list = args.skip if not skip_list: skip_list = [] logging.info("Will skip the following kinds: {0}".format(sorted(skip_list))) ds_backup = DatastoreBackup(args.app_id, zookeeper, table, source_code=args.source_code, skip_list=sorted(skip_list)) try: ds_backup.run() finally: zookeeper.close()
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations, start_gc=False) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) logging.debug("Trying to get groomer lock.") if ds_groomer.get_groomer_lock(): logging.info("Got the groomer lock.") try: ds_groomer.run_groomer() except Exception as exception: logging.exception('Encountered exception {} while running the groomer.' .format(str(exception))) try: ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH) except zk.ZKTransactionException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception))) except zk.ZKInternalException, zk_exception: logging.error("Unable to release zk lock {0}.".\ format(str(zk_exception)))
def main(): """ This main function allows you to run the restore manually. """ # Parse CLI arguments. parser = init_parser() args = parser.parse_args() # Set up logging. level = logging.INFO if args.debug: level = logging.DEBUG logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \ '%(lineno)s %(message)s ', level=level) logging.info("Logging started") logging.info(args) # Verify app is deployed. if not app_is_deployed(args.app_id): return # Verify backup dir exists. if not backup_dir_exists(args.backup_dir): return if args.clear_datastore: message = "Deleting \"{0}\" data...".\ format(args.app_id, args.backup_dir) logging.info(message) try: tables_to_clear = { APP_ENTITY_TABLE: APP_ENTITY_SCHEMA, ASC_PROPERTY_TABLE: PROPERTY_SCHEMA, DSC_PROPERTY_TABLE: PROPERTY_SCHEMA, COMPOSITE_TABLE: COMPOSITE_SCHEMA, APP_KIND_TABLE: APP_KIND_SCHEMA, TRANSACTIONS_TABLE: TRANSACTIONS_SCHEMA } for table, schema in tables_to_clear.items(): fetch_and_delete_entities('cassandra', table, schema, args.app_id, False) except Exception as exception: logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \ "Exiting...".format(args.app_id, exception.message)) return # Initialize connection to Zookeeper and database related variables. zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] # Start restore process. ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir, zookeeper, table) try: ds_restore.run() finally: zookeeper.close()
def get_datastore(): """ Returns a reference to the batch datastore interface. Validates where the <database>_interface.py is and adds that path to the system path. """ db_info = appscale_info.get_db_info() db_type = db_info[':table'] datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(db_type) return datastore_batch
def main(): """ This main function allows you to run the groomer manually. """ zookeeper = zk.ZKTransaction(host="localhost:2181") datastore_path = "localhost:8888" db_info = appscale_info.get_db_info() table = db_info[':table'] ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.run_groomer() finally: zookeeper.close()
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) datastore_path = "localhost:8888" db_info = appscale_info.get_db_info() table = db_info[":table"] ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.run_groomer() finally: zookeeper.close()
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = groomer.DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.remove_old_logs(None) finally: zookeeper.close()
def main(): """ This main function allows you to run the groomer manually. """ zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] master = appscale_info.get_db_master_ip() datastore_path = "{0}:8888".format(master) ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path) try: ds_groomer.run_groomer() finally: zookeeper.close()
def main(): """ Starts a web service for handing datastore requests. """ logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) global logger logger = logging.getLogger(__name__) global datastore_access zookeeper_locations = appscale_info.get_zk_locations_string() db_info = appscale_info.get_db_info() db_type = db_info[':table'] port = dbconstants.DEFAULT_SSL_PORT is_encrypted = True verbose = False argv = sys.argv[1:] try: opts, args = getopt.getopt( argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"]) except getopt.GetoptError: usage() sys.exit(1) for opt, arg in opts: if opt in ("-t", "--type"): db_type = arg print "Datastore type: ", db_type elif opt in ("-p", "--port"): port = int(arg) elif opt in ("-n", "--no_encryption"): is_encrypted = False elif opt in ("-v", "--verbose"): verbose = True if verbose: logger.setLevel(logging.DEBUG) if db_type not in dbconstants.VALID_DATASTORES: print "This datastore is not supported for this version of the AppScale\ datastore API:" + db_type sys.exit(1) datastore_batch = DatastoreFactory.getDatastore( db_type, log_level=logger.getEffectiveLevel()) zookeeper = zktransaction.ZKTransaction( host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel()) datastore_access = DatastoreDistributed( datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel()) if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted: port = dbconstants.DEFAULT_PORT server = tornado.httpserver.HTTPServer(pb_application) server.listen(port) while 1: try: # Start Server # tornado.ioloop.IOLoop.instance().start() except SSL.SSLError: # This happens when connections timeout, there is a just a bad # SSL connection such as it does not use SSL when expected. pass except KeyboardInterrupt: print "Server interrupted by user, terminating..." zookeeper.close() sys.exit(1)
def main(): """ Starts a web service for handing datastore requests. """ logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) global logger logger = logging.getLogger(__name__) global datastore_access zookeeper_locations = appscale_info.get_zk_locations_string() db_info = appscale_info.get_db_info() db_type = db_info[":table"] port = dbconstants.DEFAULT_SSL_PORT is_encrypted = True verbose = False argv = sys.argv[1:] try: opts, args = getopt.getopt(argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"]) except getopt.GetoptError: usage() sys.exit(1) for opt, arg in opts: if opt in ("-t", "--type"): db_type = arg print "Datastore type: ", db_type elif opt in ("-p", "--port"): port = int(arg) elif opt in ("-n", "--no_encryption"): is_encrypted = False elif opt in ("-v", "--verbose"): verbose = True if verbose: logger.setLevel(logging.DEBUG) if db_type not in dbconstants.VALID_DATASTORES: print "This datastore is not supported for this version of the AppScale\ datastore API:" + db_type sys.exit(1) datastore_batch = DatastoreFactory.getDatastore(db_type, log_level=logger.getEffectiveLevel()) zookeeper = zktransaction.ZKTransaction( host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel() ) datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel()) if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted: port = dbconstants.DEFAULT_PORT server = tornado.httpserver.HTTPServer(pb_application) server.listen(port) while 1: try: # Start Server # tornado.ioloop.IOLoop.instance().start() except SSL.SSLError: # This happens when connections timeout, there is a just a bad # SSL connection such as it does not use SSL when expected. pass except KeyboardInterrupt: print "Server interrupted by user, terminating..." zookeeper.close() sys.exit(1)
if not backup_dir_exists(args.backup_dir): return if args.clear_datastore: message = "Deleting \"{0}\" data...".\ format(args.app_id, args.backup_dir) logging.info(message) try: delete_all_records.main('cassandra', args.app_id, True) except Exception, exception: logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \ "Exiting...".format(args.app_id, exception.message)) return # Initialize connection to Zookeeper and database related variables. zk_connection_locations = appscale_info.get_zk_locations_string() zookeeper = zk.ZKTransaction(host=zk_connection_locations) db_info = appscale_info.get_db_info() table = db_info[':table'] # Start restore process. ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir, zookeeper, table) try: ds_restore.run() finally: zookeeper.close() if __name__ == "__main__": main()