def test_stop(self):
YAML_INFO = """---
:keyname: appscale
:replication: "1"
:table: cassandra
"""
flexmock(file_io).should_receive('read').and_return(YAML_INFO)
self.assertEqual('cassandra', appscale_info.get_db_info()[':table'])
self.assertEqual('1', appscale_info.get_db_info()[':replication'])
self.assertEqual('appscale', appscale_info.get_db_info()[':keyname'])
self.assertEqual(True, isinstance(appscale_info.get_db_info(), dict))
def test_stop(self):
YAML_INFO="""---
:keyname: appscale
:replication: "1"
:table: cassandra
"""
flexmock(file_io).should_receive('read').and_return(YAML_INFO)
self.assertEqual('cassandra', appscale_info.get_db_info()[':table'])
self.assertEqual( '1', appscale_info.get_db_info()[':replication'])
self.assertEqual( 'appscale', appscale_info.get_db_info()[':keyname'])
self.assertEqual(True, isinstance(appscale_info.get_db_info(), dict))
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
logging.debug("Trying to get groomer lock.")
if ds_groomer.get_groomer_lock():
logging.info("Got the groomer lock.")
try:
ds_groomer.run_groomer()
except Exception as exception:
logging.exception('Encountered exception {} while running the groomer.'
.format(str(exception)))
try:
ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
except zk.ZKTransactionException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
except zk.ZKInternalException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
def main():
""" This main function allows you to run the backup manually. """
parser = init_parser()
args = parser.parse_args()
# Set up logging.
level = logging.INFO
if args.debug:
level = logging.DEBUG
logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
'%(lineno)s %(message)s ', level=level)
logging.info("Logging started")
message = "Backing up "
if args.source_code:
message += "source and "
message += "data for: {0}".format(args.app_id)
logging.info(message)
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
skip_list = args.skip
if not skip_list:
skip_list = []
logging.info("Will skip the following kinds: {0}".format(sorted(skip_list)))
ds_backup = DatastoreBackup(args.app_id, zookeeper, table,
source_code=args.source_code, skip_list=sorted(skip_list))
try:
ds_backup.run()
finally:
zookeeper.close()
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations, start_gc=False)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
logging.debug("Trying to get groomer lock.")
if ds_groomer.get_groomer_lock():
logging.info("Got the groomer lock.")
try:
ds_groomer.run_groomer()
except Exception as exception:
logging.exception('Encountered exception {} while running the groomer.'
.format(str(exception)))
try:
ds_groomer.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
except zk.ZKTransactionException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
except zk.ZKInternalException, zk_exception:
logging.error("Unable to release zk lock {0}.".\
format(str(zk_exception)))
def main():
""" This main function allows you to run the restore manually. """
# Parse CLI arguments.
parser = init_parser()
args = parser.parse_args()
# Set up logging.
level = logging.INFO
if args.debug:
level = logging.DEBUG
logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
'%(lineno)s %(message)s ', level=level)
logging.info("Logging started")
logging.info(args)
# Verify app is deployed.
if not app_is_deployed(args.app_id):
return
# Verify backup dir exists.
if not backup_dir_exists(args.backup_dir):
return
if args.clear_datastore:
message = "Deleting \"{0}\" data...".\
format(args.app_id, args.backup_dir)
logging.info(message)
try:
tables_to_clear = {
APP_ENTITY_TABLE: APP_ENTITY_SCHEMA,
ASC_PROPERTY_TABLE: PROPERTY_SCHEMA,
DSC_PROPERTY_TABLE: PROPERTY_SCHEMA,
COMPOSITE_TABLE: COMPOSITE_SCHEMA,
APP_KIND_TABLE: APP_KIND_SCHEMA,
TRANSACTIONS_TABLE: TRANSACTIONS_SCHEMA
}
for table, schema in tables_to_clear.items():
fetch_and_delete_entities('cassandra', table, schema,
args.app_id, False)
except Exception as exception:
logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \
"Exiting...".format(args.app_id, exception.message))
return
# Initialize connection to Zookeeper and database related variables.
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
# Start restore process.
ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir,
zookeeper, table)
try:
ds_restore.run()
finally:
zookeeper.close()
def get_datastore(): """ Returns a reference to the batch datastore interface. Validates where the <database>_interface.py is and adds that path to the system path. """ db_info = appscale_info.get_db_info() db_type = db_info[':table'] datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(db_type) return datastore_batch
def get_datastore(): """ Returns a reference to the batch datastore interface. Validates where the <database>_interface.py is and adds that path to the system path. """ db_info = appscale_info.get_db_info() db_type = db_info[':table'] datastore_batch = appscale_datastore_batch.DatastoreFactory.getDatastore(db_type) return datastore_batch
def main():
""" This main function allows you to run the restore manually. """
# Parse CLI arguments.
parser = init_parser()
args = parser.parse_args()
# Set up logging.
level = logging.INFO
if args.debug:
level = logging.DEBUG
logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
'%(lineno)s %(message)s ', level=level)
logging.info("Logging started")
logging.info(args)
# Verify app is deployed.
if not app_is_deployed(args.app_id):
return
# Verify backup dir exists.
if not backup_dir_exists(args.backup_dir):
return
if args.clear_datastore:
message = "Deleting \"{0}\" data...".\
format(args.app_id, args.backup_dir)
logging.info(message)
try:
tables_to_clear = {
APP_ENTITY_TABLE: APP_ENTITY_SCHEMA,
ASC_PROPERTY_TABLE: PROPERTY_SCHEMA,
DSC_PROPERTY_TABLE: PROPERTY_SCHEMA,
COMPOSITE_TABLE: COMPOSITE_SCHEMA,
APP_KIND_TABLE: APP_KIND_SCHEMA,
TRANSACTIONS_TABLE: TRANSACTIONS_SCHEMA
}
for table, schema in tables_to_clear.items():
fetch_and_delete_entities('cassandra', table, schema, args.app_id, False)
except Exception as exception:
logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \
"Exiting...".format(args.app_id, exception.message))
return
# Initialize connection to Zookeeper and database related variables.
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
# Start restore process.
ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir,
zookeeper, table)
try:
ds_restore.run()
finally:
zookeeper.close()
def main():
""" This main function allows you to run the groomer manually. """
zookeeper = zk.ZKTransaction(host="localhost:2181")
datastore_path = "localhost:8888"
db_info = appscale_info.get_db_info()
table = db_info[':table']
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
try:
ds_groomer.run_groomer()
finally:
zookeeper.close()
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
datastore_path = "localhost:8888"
db_info = appscale_info.get_db_info()
table = db_info[":table"]
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
try:
ds_groomer.run_groomer()
finally:
zookeeper.close()
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = groomer.DatastoreGroomer(zookeeper, table, datastore_path)
try:
ds_groomer.remove_old_logs(None)
finally:
zookeeper.close()
def main():
""" This main function allows you to run the groomer manually. """
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
master = appscale_info.get_db_master_ip()
datastore_path = "{0}:8888".format(master)
ds_groomer = DatastoreGroomer(zookeeper, table, datastore_path)
try:
ds_groomer.run_groomer()
finally:
zookeeper.close()
def main():
""" Starts a web service for handing datastore requests. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
global logger
logger = logging.getLogger(__name__)
global datastore_access
zookeeper_locations = appscale_info.get_zk_locations_string()
db_info = appscale_info.get_db_info()
db_type = db_info[':table']
port = dbconstants.DEFAULT_SSL_PORT
is_encrypted = True
verbose = False
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(
argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"])
except getopt.GetoptError:
usage()
sys.exit(1)
for opt, arg in opts:
if opt in ("-t", "--type"):
db_type = arg
print "Datastore type: ", db_type
elif opt in ("-p", "--port"):
port = int(arg)
elif opt in ("-n", "--no_encryption"):
is_encrypted = False
elif opt in ("-v", "--verbose"):
verbose = True
if verbose:
logger.setLevel(logging.DEBUG)
if db_type not in dbconstants.VALID_DATASTORES:
print "This datastore is not supported for this version of the AppScale\
datastore API:" + db_type
sys.exit(1)
datastore_batch = DatastoreFactory.getDatastore(
db_type, log_level=logger.getEffectiveLevel())
zookeeper = zktransaction.ZKTransaction(
host=zookeeper_locations,
start_gc=True,
db_access=datastore_batch,
log_level=logger.getEffectiveLevel())
datastore_access = DatastoreDistributed(
datastore_batch,
zookeeper=zookeeper,
log_level=logger.getEffectiveLevel())
if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted:
port = dbconstants.DEFAULT_PORT
server = tornado.httpserver.HTTPServer(pb_application)
server.listen(port)
while 1:
try:
# Start Server #
tornado.ioloop.IOLoop.instance().start()
except SSL.SSLError:
# This happens when connections timeout, there is a just a bad
# SSL connection such as it does not use SSL when expected.
pass
except KeyboardInterrupt:
print "Server interrupted by user, terminating..."
zookeeper.close()
sys.exit(1)
def main():
""" Starts a web service for handing datastore requests. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
global logger
logger = logging.getLogger(__name__)
global datastore_access
zookeeper_locations = appscale_info.get_zk_locations_string()
db_info = appscale_info.get_db_info()
db_type = db_info[":table"]
port = dbconstants.DEFAULT_SSL_PORT
is_encrypted = True
verbose = False
argv = sys.argv[1:]
try:
opts, args = getopt.getopt(argv, "t:p:n:v:", ["type=", "port", "no_encryption", "verbose"])
except getopt.GetoptError:
usage()
sys.exit(1)
for opt, arg in opts:
if opt in ("-t", "--type"):
db_type = arg
print "Datastore type: ", db_type
elif opt in ("-p", "--port"):
port = int(arg)
elif opt in ("-n", "--no_encryption"):
is_encrypted = False
elif opt in ("-v", "--verbose"):
verbose = True
if verbose:
logger.setLevel(logging.DEBUG)
if db_type not in dbconstants.VALID_DATASTORES:
print "This datastore is not supported for this version of the AppScale\
datastore API:" + db_type
sys.exit(1)
datastore_batch = DatastoreFactory.getDatastore(db_type, log_level=logger.getEffectiveLevel())
zookeeper = zktransaction.ZKTransaction(
host=zookeeper_locations, start_gc=True, db_access=datastore_batch, log_level=logger.getEffectiveLevel()
)
datastore_access = DatastoreDistributed(datastore_batch, zookeeper=zookeeper, log_level=logger.getEffectiveLevel())
if port == dbconstants.DEFAULT_SSL_PORT and not is_encrypted:
port = dbconstants.DEFAULT_PORT
server = tornado.httpserver.HTTPServer(pb_application)
server.listen(port)
while 1:
try:
# Start Server #
tornado.ioloop.IOLoop.instance().start()
except SSL.SSLError:
# This happens when connections timeout, there is a just a bad
# SSL connection such as it does not use SSL when expected.
pass
except KeyboardInterrupt:
print "Server interrupted by user, terminating..."
zookeeper.close()
sys.exit(1)
if not backup_dir_exists(args.backup_dir):
return
if args.clear_datastore:
message = "Deleting \"{0}\" data...".\
format(args.app_id, args.backup_dir)
logging.info(message)
try:
delete_all_records.main('cassandra', args.app_id, True)
except Exception, exception:
logging.error("Unhandled exception while deleting \"{0}\" data: {1} " \
"Exiting...".format(args.app_id, exception.message))
return
# Initialize connection to Zookeeper and database related variables.
zk_connection_locations = appscale_info.get_zk_locations_string()
zookeeper = zk.ZKTransaction(host=zk_connection_locations)
db_info = appscale_info.get_db_info()
table = db_info[':table']
# Start restore process.
ds_restore = DatastoreRestore(args.app_id.strip('/'), args.backup_dir,
zookeeper, table)
try:
ds_restore.run()
finally:
zookeeper.close()
if __name__ == "__main__":
main()
