def start_cassandra(db_ips, db_master, keyname):
""" Creates a monit configuration file and prompts Monit to start Cassandra.
Args:
db_ips: A list of database node IPs to start Cassandra on.
db_master: The IP address of the DB master.
keyname: A string containing the deployment's keyname.
Raises:
AppScaleDBError if unable to start Cassandra.
"""
logging.info("Starting Cassandra...")
for ip in db_ips:
init_config = '{script} --local-ip {ip} --master-ip {db_master}'.format(
script=SETUP_CASSANDRA_SCRIPT, ip=ip, db_master=db_master)
try:
utils.ssh(ip, keyname, init_config)
except subprocess.CalledProcessError:
message = 'Unable to configure Cassandra on {}'.format(ip)
logging.exception(message)
raise dbconstants.AppScaleDBError(message)
try:
start_service_cmd = START_SERVICE_SCRIPT + CASSANDRA_WATCH_NAME
utils.ssh(ip, keyname, start_service_cmd)
except subprocess.CalledProcessError:
message = 'Unable to start Cassandra on {}'.format(ip)
logging.exception(message)
raise dbconstants.AppScaleDBError(message)
logging.info('Waiting for Cassandra to be ready')
status_cmd = '{} status'.format(cassandra_interface.NODE_TOOL)
while (utils.ssh(db_master, keyname, status_cmd, method=subprocess.call) !=
0):
time.sleep(5)
logging.info("Successfully started Cassandra.")
def wait_for_quorum(keyname, db_nodes, replication):
""" Waits until enough Cassandra nodes are up for a quorum.
Args:
keyname: A string containing the deployment's keyname.
db_nodes: An integer specifying the total number of DB nodes.
replication: An integer specifying the keyspace replication factor.
"""
command = cassandra_interface.NODE_TOOL + " " + 'status'
key_file = '{}/{}.key'.format(utils.KEY_DIRECTORY, keyname)
ssh_cmd = [
'ssh', '-i', key_file,
appscale_info.get_db_master_ip(), command
]
# Determine the number of nodes needed for a quorum.
if db_nodes < 1 or replication < 1:
raise dbconstants.AppScaleDBError(
'At least 1 database machine is needed.')
if replication > db_nodes:
raise dbconstants.AppScaleDBError(
'The replication factor cannot exceed the number of database machines.'
)
can_fail = math.ceil(replication / 2.0 - 1)
needed = int(db_nodes - can_fail)
while True:
output = subprocess.check_output(ssh_cmd)
nodes_ready = len(
[line for line in output.splitlines() if line.startswith('UN')])
logging.info('{} nodes are up. {} are needed.'.format(
nodes_ready, needed))
if nodes_ready >= needed:
break
time.sleep(1)
def estimate_total_entities(session, db_master, keyname):
""" Estimate the total number of entities.
Args:
session: A cassandra-driver session.
db_master: A string containing the IP address of the primary DB node.
keyname: A string containing the deployment keyname.
Returns:
A string containing an entity count.
Raises:
AppScaleDBError if unable to get a count.
"""
query = SimpleStatement('SELECT COUNT(*) FROM "{}"'.format(
dbconstants.APP_ENTITY_TABLE),
consistency_level=ConsistencyLevel.ONE)
try:
rows = session.execute(query)[0].count
return str(rows / len(dbconstants.APP_ENTITY_SCHEMA))
except dbconstants.TRANSIENT_CASSANDRA_ERRORS:
stats_cmd = '{nodetool} cfstats {keyspace}.{table}'.format(
nodetool=cassandra_interface.NODE_TOOL,
keyspace=cassandra_interface.KEYSPACE,
table=dbconstants.APP_ENTITY_TABLE)
stats = utils.ssh(db_master,
keyname,
stats_cmd,
method=subprocess.check_output)
for line in stats.splitlines():
if 'Number of keys (estimate)' in line:
return '{} (estimate)'.format(line.split()[-1])
raise dbconstants.AppScaleDBError('Unable to estimate total entities.')
def clean_up_kind_indices(self):
""" Deletes invalid kind index entries.
This is needed because the datastore does not delete kind index entries
when deleting entities.
"""
table_name = dbconstants.APP_KIND_TABLE
task_id = self.CLEAN_KIND_INDICES_TASK
start_key = ''
end_key = dbconstants.TERMINATING_STRING
if len(self.groomer_state) > 1:
start_key = self.groomer_state[1]
while True:
references = self.db_access.range_query(
table_name=table_name,
column_names=dbconstants.APP_KIND_SCHEMA,
start_key=start_key,
end_key=end_key,
limit=self.BATCH_SIZE,
start_inclusive=False,
)
if len(references) == 0:
break
self.index_entries_checked += len(references)
if time.time() > self.last_logged + self.LOG_PROGRESS_FREQUENCY:
logging.info('Checked {} index entries'.
format(self.index_entries_checked))
self.last_logged = time.time()
first_ref = references[0].keys()[0]
logging.debug('Fetched {} kind indices, starting with {}'.
format(len(references), [first_ref]))
last_start_key = start_key
start_key = references[-1].keys()[0]
if start_key == last_start_key:
raise dbconstants.AppScaleDBError(
'An infinite loop was detected while fetching references.')
entities = self.fetch_entity_dict_for_references(references)
for reference in references:
entity_key = reference.values()[0].values()[0]
if entity_key not in entities:
self.lock_and_delete_kind_index(reference)
self.update_groomer_state([task_id, start_key])
def clean_up_indexes(self, direction):
""" Deletes invalid single property index entries.
This is needed because we do not delete index entries when updating or
deleting entities. With time, this results in queries taking an increasing
amount of time.
Args:
direction: The direction of the index.
"""
if direction == datastore_pb.Query_Order.ASCENDING:
table_name = dbconstants.ASC_PROPERTY_TABLE
task_id = self.CLEAN_ASC_INDICES_TASK
else:
table_name = dbconstants.DSC_PROPERTY_TABLE
task_id = self.CLEAN_DSC_INDICES_TASK
# If we have state information beyond what function to use,
# load the last seen start key.
if len(self.groomer_state) > 1 and self.groomer_state[0] == task_id:
start_key = self.groomer_state[1]
else:
start_key = ''
end_key = dbconstants.TERMINATING_STRING
while True:
references = self.db_access.range_query(
table_name=table_name,
column_names=dbconstants.PROPERTY_SCHEMA,
start_key=start_key,
end_key=end_key,
limit=self.BATCH_SIZE,
start_inclusive=False,
)
if len(references) == 0:
break
self.index_entries_checked += len(references)
if time.time() > self.last_logged + self.LOG_PROGRESS_FREQUENCY:
logging.info('Checked {} index entries'
.format(self.index_entries_checked))
self.last_logged = time.time()
first_ref = references[0].keys()[0]
logging.debug('Fetched {} total refs, starting with {}, direction: {}'
.format(self.index_entries_checked, [first_ref], direction))
last_start_key = start_key
start_key = references[-1].keys()[0]
if start_key == last_start_key:
raise dbconstants.AppScaleDBError(
'An infinite loop was detected while fetching references.')
entities = self.fetch_entity_dict_for_references(references)
# Group invalid references by entity key so we can minimize locks.
invalid_refs = {}
for reference in references:
prop_name = reference.keys()[0].split(self.ds_access._SEPARATOR)[3]
if not self.ds_access._DatastoreDistributed__valid_index_entry(
reference, entities, direction, prop_name):
entity_key = reference.values()[0][self.ds_access.INDEX_REFERENCE_COLUMN]
if entity_key not in invalid_refs:
invalid_refs[entity_key] = []
invalid_refs[entity_key].append(reference)
for entity_key in invalid_refs:
self.lock_and_delete_indexes(invalid_refs[entity_key], direction, entity_key)
self.update_groomer_state([task_id, start_key])