def restore_data(path, keyname, force=False):
""" Restores the Cassandra backup.
Args:
path: A string containing the location on each of the DB machines to use
for restoring data.
keyname: A string containing the deployment's keyname.
Raises:
BRException if unable to find any Cassandra machines or if DB machine has
insufficient space.
"""
logging.info("Starting new db restore.")
db_ips = appscale_info.get_db_ips()
if not db_ips:
raise BRException("Unable to find any Cassandra machines.")
machines_without_restore = []
for db_ip in db_ips:
exit_code = utils.ssh(db_ip, keyname, "ls {}".format(path), method=subprocess.call)
if exit_code != ExitCodes.SUCCESS:
machines_without_restore.append(db_ip)
if machines_without_restore and not force:
logging.info("The following machines do not have a restore file: {}".format(machines_without_restore))
response = raw_input("Would you like to continue? [y/N] ")
if response not in ["Y", "y"]:
return
for db_ip in db_ips:
logging.info("Stopping Cassandra on {}".format(db_ip))
summary = utils.ssh(db_ip, keyname, "monit summary", method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries = SERVICE_STOP_RETRIES
while status != MonitStates.UNMONITORED:
utils.ssh(db_ip, keyname, "monit stop {}".format(CASSANDRA_MONIT_WATCH_NAME), method=subprocess.call)
time.sleep(3)
summary = utils.ssh(db_ip, keyname, "monit summary", method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries -= 1
if retries < 0:
raise BRException("Unable to stop Cassandra")
cassandra_dir = "{}/cassandra".format(APPSCALE_DATA_DIR)
for db_ip in db_ips:
logging.info("Restoring Cassandra data on {}".format(db_ip))
clear_db = 'find {0} -regex ".*\.\(db\|txt\|log\)$" -exec rm {{}} \;'.format(cassandra_dir)
utils.ssh(db_ip, keyname, clear_db)
if db_ip not in machines_without_restore:
utils.ssh(db_ip, keyname, "tar xf {} -C {}".format(path, cassandra_dir))
utils.ssh(db_ip, keyname, "chown -R cassandra {}".format(cassandra_dir))
utils.ssh(db_ip, keyname, "monit start {}".format(CASSANDRA_MONIT_WATCH_NAME))
logging.info("Done with db restore.")
def restore_data(path, keyname, force=False):
""" Restores the Cassandra backup.
Args:
path: A string containing the location on each of the DB machines to use
for restoring data.
keyname: A string containing the deployment's keyname.
Raises:
BRException if unable to find any Cassandra machines or if DB machine has
insufficient space.
"""
logging.info("Starting new db restore.")
db_ips = appscale_info.get_db_ips()
if not db_ips:
raise BRException('Unable to find any Cassandra machines.')
machines_without_restore = []
for db_ip in db_ips:
exit_code = utils.ssh(db_ip,
keyname,
'ls {}'.format(path),
method=subprocess.call)
if exit_code != ExitCodes.SUCCESS:
machines_without_restore.append(db_ip)
if machines_without_restore and not force:
logging.info(
'The following machines do not have a restore file: {}'.format(
machines_without_restore))
response = raw_input('Would you like to continue? [y/N] ')
if response not in ['Y', 'y']:
return
for db_ip in db_ips:
logging.info('Stopping Cassandra on {}'.format(db_ip))
summary = utils.ssh(db_ip,
keyname,
'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries = SERVICE_STOP_RETRIES
while status != MonitStates.UNMONITORED:
utils.ssh(db_ip, keyname,
'monit stop {}'.format(CASSANDRA_MONIT_WATCH_NAME))
time.sleep(1)
summary = utils.ssh(db_ip,
keyname,
'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries -= 1
if retries < 0:
raise BRException('Unable to stop Cassandra')
cassandra_dir = '{}/cassandra'.format(APPSCALE_DATA_DIR)
for db_ip in db_ips:
logging.info('Restoring Cassandra data on {}'.format(db_ip))
clear_db = 'find {0} -regex ".*\.\(db\|txt\|log\)$" -exec rm {{}} \;'.\
format(cassandra_dir)
utils.ssh(db_ip, keyname, clear_db)
if db_ip not in machines_without_restore:
utils.ssh(db_ip, keyname,
'tar xf {} -C {}'.format(path, cassandra_dir))
utils.ssh(db_ip, keyname,
'monit start {}'.format(CASSANDRA_MONIT_WATCH_NAME))
logging.info("Done with db restore.")
def restore_data(path, keyname, force=False):
""" Restores the Cassandra backup.
Args:
path: A string containing the location on each of the DB machines to use
for restoring data.
keyname: A string containing the deployment's keyname.
Raises:
BRException if unable to find any Cassandra machines or if DB machine has
insufficient space.
"""
logging.info("Starting new db restore.")
db_ips = appscale_info.get_db_ips()
if not db_ips:
raise BRException('Unable to find any Cassandra machines.')
machines_without_restore = []
for db_ip in db_ips:
exit_code = appscale_utils.ssh(db_ip,
keyname,
'ls {}'.format(path),
method=subprocess.call)
if exit_code != ExitCodes.SUCCESS:
machines_without_restore.append(db_ip)
if machines_without_restore and not force:
logging.info(
'The following machines do not have a restore file: {}'.format(
machines_without_restore))
response = raw_input('Would you like to continue? [y/N] ')
if response not in ['Y', 'y']:
return
for db_ip in db_ips:
logging.info('Stopping Cassandra on {}'.format(db_ip))
summary = appscale_utils.ssh(db_ip,
keyname,
'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries = SERVICE_RETRIES
while status != MonitStates.UNMONITORED:
appscale_utils.ssh(
db_ip,
keyname,
'appscale-stop-service {}'.format(CASSANDRA_MONIT_WATCH_NAME),
method=subprocess.call)
time.sleep(3)
summary = appscale_utils.ssh(db_ip,
keyname,
'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries -= 1
if retries < 0:
raise BRException('Unable to stop Cassandra')
cassandra_dir = '{}/cassandra'.format(APPSCALE_DATA_DIR)
for db_ip in db_ips:
logging.info('Restoring Cassandra data on {}'.format(db_ip))
clear_db = 'find {0} -regex ".*\.\(db\|txt\|log\)$" -exec rm {{}} \;'.\
format(cassandra_dir)
appscale_utils.ssh(db_ip, keyname, clear_db)
if db_ip not in machines_without_restore:
appscale_utils.ssh(db_ip, keyname,
'tar xf {} -C {}'.format(path, cassandra_dir))
appscale_utils.ssh(db_ip, keyname,
'chown -R cassandra {}'.format(cassandra_dir))
logging.info('Starting Cassandra on {}'.format(db_ip))
retries = SERVICE_RETRIES
status = MonitStates.UNMONITORED
while status != MonitStates.RUNNING:
appscale_utils.ssh(
db_ip,
keyname,
'appscale-start-service {}'.format(CASSANDRA_MONIT_WATCH_NAME),
method=subprocess.call)
time.sleep(3)
summary = appscale_utils.ssh(db_ip,
keyname,
'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries -= 1
if retries < 0:
raise BRException('Unable to start Cassandra')
appscale_utils.ssh(
db_ip, keyname,
'appscale-start-service {}'.format(CASSANDRA_MONIT_WATCH_NAME))
logging.info('Waiting for Cassandra cluster to be ready')
db_ip = db_ips[0]
deadline = time.time() + SCHEMA_CHANGE_TIMEOUT
while True:
ready = True
try:
output = appscale_utils.ssh(db_ip,
keyname,
'{} status'.format(NODE_TOOL),
method=subprocess.check_output)
nodes_ready = len(
[line for line in output.split('\n') if line.startswith('UN')])
if nodes_ready < len(db_ips):
ready = False
except CalledProcessError:
ready = False
if ready:
break
if time.time() > deadline:
logging.warning('Cassandra cluster still not ready.')
break
time.sleep(3)
logging.info("Done with db restore.")
def restore_data(path, keyname, force=False):
""" Restores the Cassandra backup.
Args:
path: A string containing the location on each of the DB machines to use
for restoring data.
keyname: A string containing the deployment's keyname.
Raises:
BRException if unable to find any Cassandra machines or if DB machine has
insufficient space.
"""
logging.info("Starting new db restore.")
db_ips = appscale_info.get_db_ips()
if not db_ips:
raise BRException('Unable to find any Cassandra machines.')
machines_without_restore = []
for db_ip in db_ips:
exit_code = appscale_utils.ssh(db_ip, keyname, 'ls {}'.format(path),
method=subprocess.call)
if exit_code != ExitCodes.SUCCESS:
machines_without_restore.append(db_ip)
if machines_without_restore and not force:
logging.info('The following machines do not have a restore file: {}'.
format(machines_without_restore))
response = raw_input('Would you like to continue? [y/N] ')
if response not in ['Y', 'y']:
return
for db_ip in db_ips:
logging.info('Stopping Cassandra on {}'.format(db_ip))
summary = appscale_utils.ssh(db_ip, keyname, 'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries = SERVICE_RETRIES
while status != MonitStates.UNMONITORED:
appscale_utils.ssh(db_ip, keyname,
'monit stop {}'.format(CASSANDRA_MONIT_WATCH_NAME),
method=subprocess.call)
time.sleep(3)
summary = appscale_utils.ssh(db_ip, keyname, 'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries -= 1
if retries < 0:
raise BRException('Unable to stop Cassandra')
cassandra_dir = '{}/cassandra'.format(APPSCALE_DATA_DIR)
for db_ip in db_ips:
logging.info('Restoring Cassandra data on {}'.format(db_ip))
clear_db = 'find {0} -regex ".*\.\(db\|txt\|log\)$" -exec rm {{}} \;'.\
format(cassandra_dir)
appscale_utils.ssh(db_ip, keyname, clear_db)
if db_ip not in machines_without_restore:
appscale_utils.ssh(db_ip, keyname,
'tar xf {} -C {}'.format(path, cassandra_dir))
appscale_utils.ssh(db_ip, keyname,
'chown -R cassandra {}'.format(cassandra_dir))
logging.info('Starting Cassandra on {}'.format(db_ip))
retries = SERVICE_RETRIES
status = MonitStates.UNMONITORED
while status != MonitStates.RUNNING:
appscale_utils.ssh(db_ip, keyname,
'monit start {}'.format(CASSANDRA_MONIT_WATCH_NAME),
method=subprocess.call)
time.sleep(3)
summary = appscale_utils.ssh(db_ip, keyname, 'monit summary',
method=subprocess.check_output)
status = utils.monit_status(summary, CASSANDRA_MONIT_WATCH_NAME)
retries -= 1
if retries < 0:
raise BRException('Unable to start Cassandra')
appscale_utils.ssh(db_ip, keyname,
'monit start {}'.format(CASSANDRA_MONIT_WATCH_NAME))
logging.info('Waiting for Cassandra cluster to be ready')
db_ip = db_ips[0]
deadline = time.time() + SCHEMA_CHANGE_TIMEOUT
while True:
ready = True
try:
output = appscale_utils.ssh(
db_ip, keyname, '{} status'.format(NODE_TOOL),
method=subprocess.check_output)
nodes_ready = len([line for line in output.split('\n')
if line.startswith('UN')])
if nodes_ready < len(db_ips):
ready = False
except CalledProcessError:
ready = False
if ready:
break
if time.time() > deadline:
logging.warning('Cassandra cluster still not ready.')
break
time.sleep(3)
logging.info("Done with db restore.")