def prime_cassandra(replication):
""" Create Cassandra keyspace and initial tables.
Args:
replication: An integer specifying the replication factor for the keyspace.
Raises:
AppScaleBadArg if replication factor is not greater than 0.
TypeError if replication is not an integer.
"""
if not isinstance(replication, int):
raise TypeError('Replication must be an integer')
if int(replication) <= 0:
raise dbconstants.AppScaleBadArg('Replication must be greater than zero')
hosts = appscale_info.get_db_ips()
cluster = None
session = None
remaining_retries = INITIAL_CONNECT_RETRIES
while True:
try:
cluster = Cluster(hosts)
session = cluster.connect()
break
except cassandra.cluster.NoHostAvailable as connection_error:
remaining_retries -= 1
if remaining_retries < 0:
raise connection_error
time.sleep(3)
session.default_consistency_level = ConsistencyLevel.QUORUM
create_keyspace = """
CREATE KEYSPACE IF NOT EXISTS "{keyspace}"
WITH REPLICATION = %(replication)s
""".format(keyspace=KEYSPACE)
keyspace_replication = {'class': 'SimpleStrategy',
'replication_factor': replication}
session.execute(create_keyspace, {'replication': keyspace_replication})
session.set_keyspace(KEYSPACE)
for table in dbconstants.INITIAL_TABLES:
create_table = """
CREATE TABLE IF NOT EXISTS "{table}" (
{key} blob,
{column} text,
{value} blob,
PRIMARY KEY ({key}, {column})
) WITH COMPACT STORAGE
""".format(table=table,
key=ThriftColumn.KEY,
column=ThriftColumn.COLUMN_NAME,
value=ThriftColumn.VALUE)
statement = SimpleStatement(create_table, retry_policy=NO_RETRIES)
logging.info('Trying to create {}'.format(table))
try:
session.execute(statement)
except cassandra.OperationTimedOut:
logging.warning(
'Encountered an operation timeout while creating {} table. '
'Waiting 1 minute for schema to settle.'.format(table))
time.sleep(60)
raise
create_batch_tables(cluster, session)
create_pull_queue_tables(cluster, session)
first_entity = session.execute(
'SELECT * FROM "{}" LIMIT 1'.format(dbconstants.APP_ENTITY_TABLE))
existing_entities = len(list(first_entity)) == 1
define_ua_schema(session)
metadata_insert = """
INSERT INTO "{table}" ({key}, {column}, {value})
VALUES (%(key)s, %(column)s, %(value)s)
""".format(
table=dbconstants.DATASTORE_METADATA_TABLE,
key=ThriftColumn.KEY,
column=ThriftColumn.COLUMN_NAME,
value=ThriftColumn.VALUE
)
if not existing_entities:
parameters = {'key': bytearray(cassandra_interface.VERSION_INFO_KEY),
'column': cassandra_interface.VERSION_INFO_KEY,
'value': bytearray(str(POST_JOURNAL_VERSION))}
session.execute(metadata_insert, parameters)
# Indicate that the database has been successfully primed.
parameters = {'key': bytearray(cassandra_interface.PRIMED_KEY),
'column': cassandra_interface.PRIMED_KEY,
'value': bytearray('true')}
session.execute(metadata_insert, parameters)
logging.info('Cassandra is primed.')
def prime_cassandra(replication):
""" Create Cassandra keyspace and initial tables.
Args:
replication: An integer specifying the replication factor for the keyspace.
Raises:
AppScaleBadArg if replication factor is not greater than 0.
TypeError if replication is not an integer.
"""
if not isinstance(replication, int):
raise TypeError('Replication must be an integer')
if int(replication) <= 0:
raise dbconstants.AppScaleBadArg(
'Replication must be greater than zero')
hosts = appscale_info.get_db_ips()
cluster = None
session = None
remaining_retries = INITIAL_CONNECT_RETRIES
while True:
try:
cluster = Cluster(hosts)
session = cluster.connect()
break
except cassandra.cluster.NoHostAvailable as connection_error:
remaining_retries -= 1
if remaining_retries < 0:
raise connection_error
time.sleep(3)
session.default_consistency_level = ConsistencyLevel.QUORUM
create_keyspace = """
CREATE KEYSPACE IF NOT EXISTS "{keyspace}"
WITH REPLICATION = %(replication)s
""".format(keyspace=KEYSPACE)
keyspace_replication = {
'class': 'SimpleStrategy',
'replication_factor': replication
}
session.execute(create_keyspace, {'replication': keyspace_replication})
session.set_keyspace(KEYSPACE)
for table in dbconstants.INITIAL_TABLES:
create_table = """
CREATE TABLE IF NOT EXISTS "{table}" (
{key} blob,
{column} text,
{value} blob,
PRIMARY KEY ({key}, {column})
) WITH COMPACT STORAGE
""".format(table=table,
key=ThriftColumn.KEY,
column=ThriftColumn.COLUMN_NAME,
value=ThriftColumn.VALUE)
statement = SimpleStatement(create_table, retry_policy=NO_RETRIES)
logging.info('Trying to create {}'.format(table))
try:
session.execute(statement)
except cassandra.OperationTimedOut:
logging.warning(
'Encountered an operation timeout while creating {} table. '
'Waiting 1 minute for schema to settle.'.format(table))
time.sleep(60)
raise
create_batch_tables(cluster, session)
create_groups_table(session)
create_transactions_table(session)
create_pull_queue_tables(cluster, session)
first_entity = session.execute('SELECT * FROM "{}" LIMIT 1'.format(
dbconstants.APP_ENTITY_TABLE))
existing_entities = len(list(first_entity)) == 1
define_ua_schema(session)
metadata_insert = """
INSERT INTO "{table}" ({key}, {column}, {value})
VALUES (%(key)s, %(column)s, %(value)s)
""".format(table=dbconstants.DATASTORE_METADATA_TABLE,
key=ThriftColumn.KEY,
column=ThriftColumn.COLUMN_NAME,
value=ThriftColumn.VALUE)
if not existing_entities:
parameters = {
'key': bytearray(cassandra_interface.VERSION_INFO_KEY),
'column': cassandra_interface.VERSION_INFO_KEY,
'value': bytearray(str(POST_JOURNAL_VERSION))
}
session.execute(metadata_insert, parameters)
# Mark the newly created indexes as clean.
parameters = {
'key': bytearray(cassandra_interface.INDEX_STATE_KEY),
'column': cassandra_interface.INDEX_STATE_KEY,
'value': bytearray(str(IndexStates.CLEAN))
}
session.execute(metadata_insert, parameters)
# Indicate that the database has been successfully primed.
parameters = {
'key': bytearray(cassandra_interface.PRIMED_KEY),
'column': cassandra_interface.PRIMED_KEY,
'value': bytearray('true')
}
session.execute(metadata_insert, parameters)
logging.info('Cassandra is primed.')
def prime_cassandra(replication):
""" Create Cassandra keyspace and initial tables.
Args:
replication: An integer specifying the replication factor for the keyspace.
Raises:
AppScaleBadArg if replication factor is not greater than 0.
TypeError if replication is not an integer.
"""
if not isinstance(replication, int):
raise TypeError('Replication must be an integer')
if int(replication) <= 0:
raise dbconstants.AppScaleBadArg('Replication must be greater than zero')
zk_client = KazooClient(hosts=appscale_info.get_zk_node_ips())
zk_client.start()
hosts = appscale_info.get_db_ips()
remaining_retries = INITIAL_CONNECT_RETRIES
while True:
try:
cluster = Cluster(hosts, load_balancing_policy=LB_POLICY)
session = cluster.connect()
break
except cassandra.cluster.NoHostAvailable as connection_error:
remaining_retries -= 1
if remaining_retries < 0:
raise connection_error
time.sleep(3)
session.default_consistency_level = ConsistencyLevel.QUORUM
create_keyspace = """
CREATE KEYSPACE IF NOT EXISTS "{keyspace}"
WITH REPLICATION = %(replication)s
""".format(keyspace=KEYSPACE)
keyspace_replication = {'class': 'SimpleStrategy',
'replication_factor': replication}
session.execute(create_keyspace, {'replication': keyspace_replication},
timeout=SCHEMA_CHANGE_TIMEOUT)
session.set_keyspace(KEYSPACE)
logger.info('Waiting for all hosts to be connected')
deadline = time.time() + SCHEMA_CHANGE_TIMEOUT
while True:
if time.time() > deadline:
logger.warning('Timeout when waiting for hosts to join. Continuing '
'with connected hosts.')
break
if len(session.get_pool_state()) == len(hosts):
break
time.sleep(1)
for table in dbconstants.INITIAL_TABLES:
create_table = """
CREATE TABLE IF NOT EXISTS "{table}" (
{key} blob,
{column} text,
{value} blob,
PRIMARY KEY ({key}, {column})
) WITH COMPACT STORAGE
""".format(table=table,
key=ThriftColumn.KEY,
column=ThriftColumn.COLUMN_NAME,
value=ThriftColumn.VALUE)
statement = SimpleStatement(create_table, retry_policy=NO_RETRIES)
logger.info('Trying to create {}'.format(table))
try:
session.execute(statement, timeout=SCHEMA_CHANGE_TIMEOUT)
except cassandra.OperationTimedOut:
logger.warning(
'Encountered an operation timeout while creating {} table. Waiting {} '
'seconds for schema to settle.'.format(table, SCHEMA_CHANGE_TIMEOUT))
time.sleep(SCHEMA_CHANGE_TIMEOUT)
raise
migrate_composite_index_metadata(cluster, session, zk_client)
create_batch_tables(cluster, session)
create_groups_table(session)
create_transactions_table(session)
create_pull_queue_tables(cluster, session)
create_entity_ids_table(session)
first_entity = session.execute(
'SELECT * FROM "{}" LIMIT 1'.format(dbconstants.APP_ENTITY_TABLE))
existing_entities = len(list(first_entity)) == 1
define_ua_schema(session)
metadata_insert = """
INSERT INTO "{table}" ({key}, {column}, {value})
VALUES (%(key)s, %(column)s, %(value)s)
""".format(
table=dbconstants.DATASTORE_METADATA_TABLE,
key=ThriftColumn.KEY,
column=ThriftColumn.COLUMN_NAME,
value=ThriftColumn.VALUE
)
if existing_entities:
current_version = current_datastore_version(session)
if current_version == 1.0:
# Instruct the groomer to reclean the indexes.
parameters = {'key': bytearray(cassandra_interface.INDEX_STATE_KEY),
'column': cassandra_interface.INDEX_STATE_KEY,
'value': bytearray(str(IndexStates.DIRTY))}
session.execute(metadata_insert, parameters)
parameters = {'key': bytearray(cassandra_interface.VERSION_INFO_KEY),
'column': cassandra_interface.VERSION_INFO_KEY,
'value': bytearray(str(CURRENT_VERSION))}
session.execute(metadata_insert, parameters)
else:
parameters = {'key': bytearray(cassandra_interface.VERSION_INFO_KEY),
'column': cassandra_interface.VERSION_INFO_KEY,
'value': bytearray(str(CURRENT_VERSION))}
session.execute(metadata_insert, parameters)
# Mark the newly created indexes as clean.
parameters = {'key': bytearray(cassandra_interface.INDEX_STATE_KEY),
'column': cassandra_interface.INDEX_STATE_KEY,
'value': bytearray(str(IndexStates.CLEAN))}
session.execute(metadata_insert, parameters)
# Indicate that scatter property values do not need to be populated.
parameters = {'key': bytearray(cassandra_interface.SCATTER_PROP_KEY),
'column': cassandra_interface.SCATTER_PROP_KEY,
'value': bytearray(ScatterPropStates.POPULATED)}
session.execute(metadata_insert, parameters)
# Indicate that the database has been successfully primed.
parameters = {'key': bytearray(cassandra_interface.PRIMED_KEY),
'column': cassandra_interface.PRIMED_KEY,
'value': bytearray(str(CURRENT_VERSION))}
session.execute(metadata_insert, parameters)
logger.info('Cassandra is primed.')