def prime_cassandra(replication): """ Create Cassandra keyspace and initial tables. Args: replication: An integer specifying the replication factor for the keyspace. Raises: AppScaleBadArg if replication factor is not greater than 0. TypeError if replication is not an integer. """ if not isinstance(replication, int): raise TypeError('Replication must be an integer') if int(replication) <= 0: raise dbconstants.AppScaleBadArg('Replication must be greater than zero') hosts = appscale_info.get_db_ips() cluster = None session = None remaining_retries = INITIAL_CONNECT_RETRIES while True: try: cluster = Cluster(hosts) session = cluster.connect() break except cassandra.cluster.NoHostAvailable as connection_error: remaining_retries -= 1 if remaining_retries < 0: raise connection_error time.sleep(3) session.default_consistency_level = ConsistencyLevel.QUORUM create_keyspace = """ CREATE KEYSPACE IF NOT EXISTS "{keyspace}" WITH REPLICATION = %(replication)s """.format(keyspace=KEYSPACE) keyspace_replication = {'class': 'SimpleStrategy', 'replication_factor': replication} session.execute(create_keyspace, {'replication': keyspace_replication}) session.set_keyspace(KEYSPACE) for table in dbconstants.INITIAL_TABLES: create_table = """ CREATE TABLE IF NOT EXISTS "{table}" ( {key} blob, {column} text, {value} blob, PRIMARY KEY ({key}, {column}) ) WITH COMPACT STORAGE """.format(table=table, key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME, value=ThriftColumn.VALUE) statement = SimpleStatement(create_table, retry_policy=NO_RETRIES) logging.info('Trying to create {}'.format(table)) try: session.execute(statement) except cassandra.OperationTimedOut: logging.warning( 'Encountered an operation timeout while creating {} table. ' 'Waiting 1 minute for schema to settle.'.format(table)) time.sleep(60) raise create_batch_tables(cluster, session) create_pull_queue_tables(cluster, session) first_entity = session.execute( 'SELECT * FROM "{}" LIMIT 1'.format(dbconstants.APP_ENTITY_TABLE)) existing_entities = len(list(first_entity)) == 1 define_ua_schema(session) metadata_insert = """ INSERT INTO "{table}" ({key}, {column}, {value}) VALUES (%(key)s, %(column)s, %(value)s) """.format( table=dbconstants.DATASTORE_METADATA_TABLE, key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME, value=ThriftColumn.VALUE ) if not existing_entities: parameters = {'key': bytearray(cassandra_interface.VERSION_INFO_KEY), 'column': cassandra_interface.VERSION_INFO_KEY, 'value': bytearray(str(POST_JOURNAL_VERSION))} session.execute(metadata_insert, parameters) # Indicate that the database has been successfully primed. parameters = {'key': bytearray(cassandra_interface.PRIMED_KEY), 'column': cassandra_interface.PRIMED_KEY, 'value': bytearray('true')} session.execute(metadata_insert, parameters) logging.info('Cassandra is primed.')
def prime_cassandra(replication): """ Create Cassandra keyspace and initial tables. Args: replication: An integer specifying the replication factor for the keyspace. Raises: AppScaleBadArg if replication factor is not greater than 0. TypeError if replication is not an integer. """ if not isinstance(replication, int): raise TypeError('Replication must be an integer') if int(replication) <= 0: raise dbconstants.AppScaleBadArg( 'Replication must be greater than zero') hosts = appscale_info.get_db_ips() cluster = None session = None remaining_retries = INITIAL_CONNECT_RETRIES while True: try: cluster = Cluster(hosts) session = cluster.connect() break except cassandra.cluster.NoHostAvailable as connection_error: remaining_retries -= 1 if remaining_retries < 0: raise connection_error time.sleep(3) session.default_consistency_level = ConsistencyLevel.QUORUM create_keyspace = """ CREATE KEYSPACE IF NOT EXISTS "{keyspace}" WITH REPLICATION = %(replication)s """.format(keyspace=KEYSPACE) keyspace_replication = { 'class': 'SimpleStrategy', 'replication_factor': replication } session.execute(create_keyspace, {'replication': keyspace_replication}) session.set_keyspace(KEYSPACE) for table in dbconstants.INITIAL_TABLES: create_table = """ CREATE TABLE IF NOT EXISTS "{table}" ( {key} blob, {column} text, {value} blob, PRIMARY KEY ({key}, {column}) ) WITH COMPACT STORAGE """.format(table=table, key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME, value=ThriftColumn.VALUE) statement = SimpleStatement(create_table, retry_policy=NO_RETRIES) logging.info('Trying to create {}'.format(table)) try: session.execute(statement) except cassandra.OperationTimedOut: logging.warning( 'Encountered an operation timeout while creating {} table. ' 'Waiting 1 minute for schema to settle.'.format(table)) time.sleep(60) raise create_batch_tables(cluster, session) create_groups_table(session) create_transactions_table(session) create_pull_queue_tables(cluster, session) first_entity = session.execute('SELECT * FROM "{}" LIMIT 1'.format( dbconstants.APP_ENTITY_TABLE)) existing_entities = len(list(first_entity)) == 1 define_ua_schema(session) metadata_insert = """ INSERT INTO "{table}" ({key}, {column}, {value}) VALUES (%(key)s, %(column)s, %(value)s) """.format(table=dbconstants.DATASTORE_METADATA_TABLE, key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME, value=ThriftColumn.VALUE) if not existing_entities: parameters = { 'key': bytearray(cassandra_interface.VERSION_INFO_KEY), 'column': cassandra_interface.VERSION_INFO_KEY, 'value': bytearray(str(POST_JOURNAL_VERSION)) } session.execute(metadata_insert, parameters) # Mark the newly created indexes as clean. parameters = { 'key': bytearray(cassandra_interface.INDEX_STATE_KEY), 'column': cassandra_interface.INDEX_STATE_KEY, 'value': bytearray(str(IndexStates.CLEAN)) } session.execute(metadata_insert, parameters) # Indicate that the database has been successfully primed. parameters = { 'key': bytearray(cassandra_interface.PRIMED_KEY), 'column': cassandra_interface.PRIMED_KEY, 'value': bytearray('true') } session.execute(metadata_insert, parameters) logging.info('Cassandra is primed.')
def prime_cassandra(replication): """ Create Cassandra keyspace and initial tables. Args: replication: An integer specifying the replication factor for the keyspace. Raises: AppScaleBadArg if replication factor is not greater than 0. TypeError if replication is not an integer. """ if not isinstance(replication, int): raise TypeError('Replication must be an integer') if int(replication) <= 0: raise dbconstants.AppScaleBadArg('Replication must be greater than zero') zk_client = KazooClient(hosts=appscale_info.get_zk_node_ips()) zk_client.start() hosts = appscale_info.get_db_ips() remaining_retries = INITIAL_CONNECT_RETRIES while True: try: cluster = Cluster(hosts, load_balancing_policy=LB_POLICY) session = cluster.connect() break except cassandra.cluster.NoHostAvailable as connection_error: remaining_retries -= 1 if remaining_retries < 0: raise connection_error time.sleep(3) session.default_consistency_level = ConsistencyLevel.QUORUM create_keyspace = """ CREATE KEYSPACE IF NOT EXISTS "{keyspace}" WITH REPLICATION = %(replication)s """.format(keyspace=KEYSPACE) keyspace_replication = {'class': 'SimpleStrategy', 'replication_factor': replication} session.execute(create_keyspace, {'replication': keyspace_replication}, timeout=SCHEMA_CHANGE_TIMEOUT) session.set_keyspace(KEYSPACE) logger.info('Waiting for all hosts to be connected') deadline = time.time() + SCHEMA_CHANGE_TIMEOUT while True: if time.time() > deadline: logger.warning('Timeout when waiting for hosts to join. Continuing ' 'with connected hosts.') break if len(session.get_pool_state()) == len(hosts): break time.sleep(1) for table in dbconstants.INITIAL_TABLES: create_table = """ CREATE TABLE IF NOT EXISTS "{table}" ( {key} blob, {column} text, {value} blob, PRIMARY KEY ({key}, {column}) ) WITH COMPACT STORAGE """.format(table=table, key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME, value=ThriftColumn.VALUE) statement = SimpleStatement(create_table, retry_policy=NO_RETRIES) logger.info('Trying to create {}'.format(table)) try: session.execute(statement, timeout=SCHEMA_CHANGE_TIMEOUT) except cassandra.OperationTimedOut: logger.warning( 'Encountered an operation timeout while creating {} table. Waiting {} ' 'seconds for schema to settle.'.format(table, SCHEMA_CHANGE_TIMEOUT)) time.sleep(SCHEMA_CHANGE_TIMEOUT) raise migrate_composite_index_metadata(cluster, session, zk_client) create_batch_tables(cluster, session) create_groups_table(session) create_transactions_table(session) create_pull_queue_tables(cluster, session) create_entity_ids_table(session) first_entity = session.execute( 'SELECT * FROM "{}" LIMIT 1'.format(dbconstants.APP_ENTITY_TABLE)) existing_entities = len(list(first_entity)) == 1 define_ua_schema(session) metadata_insert = """ INSERT INTO "{table}" ({key}, {column}, {value}) VALUES (%(key)s, %(column)s, %(value)s) """.format( table=dbconstants.DATASTORE_METADATA_TABLE, key=ThriftColumn.KEY, column=ThriftColumn.COLUMN_NAME, value=ThriftColumn.VALUE ) if existing_entities: current_version = current_datastore_version(session) if current_version == 1.0: # Instruct the groomer to reclean the indexes. parameters = {'key': bytearray(cassandra_interface.INDEX_STATE_KEY), 'column': cassandra_interface.INDEX_STATE_KEY, 'value': bytearray(str(IndexStates.DIRTY))} session.execute(metadata_insert, parameters) parameters = {'key': bytearray(cassandra_interface.VERSION_INFO_KEY), 'column': cassandra_interface.VERSION_INFO_KEY, 'value': bytearray(str(CURRENT_VERSION))} session.execute(metadata_insert, parameters) else: parameters = {'key': bytearray(cassandra_interface.VERSION_INFO_KEY), 'column': cassandra_interface.VERSION_INFO_KEY, 'value': bytearray(str(CURRENT_VERSION))} session.execute(metadata_insert, parameters) # Mark the newly created indexes as clean. parameters = {'key': bytearray(cassandra_interface.INDEX_STATE_KEY), 'column': cassandra_interface.INDEX_STATE_KEY, 'value': bytearray(str(IndexStates.CLEAN))} session.execute(metadata_insert, parameters) # Indicate that scatter property values do not need to be populated. parameters = {'key': bytearray(cassandra_interface.SCATTER_PROP_KEY), 'column': cassandra_interface.SCATTER_PROP_KEY, 'value': bytearray(ScatterPropStates.POPULATED)} session.execute(metadata_insert, parameters) # Indicate that the database has been successfully primed. parameters = {'key': bytearray(cassandra_interface.PRIMED_KEY), 'column': cassandra_interface.PRIMED_KEY, 'value': bytearray(str(CURRENT_VERSION))} session.execute(metadata_insert, parameters) logger.info('Cassandra is primed.')