def _test_turnoff_translog_retention_after_upgraded(self, path): cluster = self._new_cluster(path.from_version, self.NUMBER_OF_NODES) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() number_of_replicas = random.randint(0, 2) c.execute( ''' create table doc.test(x int) clustered into 1 shards with( number_of_replicas =?, "soft_deletes.enabled" = true) ''', (number_of_replicas, )) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) insert_data(conn, 'doc', 'test', random.randint(100, 200)) c.execute('refresh table doc.test') num_docs = random.randint(0, 100) if num_docs > 0: insert_data(conn, 'doc', 'test', num_docs) # update the cluster to the new version self._upgrade_cluster(cluster, path.to_version, self.NUMBER_OF_NODES) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) c.execute('refresh table doc.test') self._assert_translog_is_empty(conn, 'doc', 'test')
def _test_upgrade_path(self, versions: Tuple[VersionDef], nodes): """ Test upgrade path across specified versions. Creates a blob and regular table in first version and inserts a record, then goes through all subsequent versions - each time verifying that a few simple selects work. """ version_def = versions[0] env = prepare_env(version_def.java_home) cluster = self._new_cluster(version_def.version, nodes, self.CLUSTER_SETTINGS, env) cluster.start() digest = None with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(CREATE_ANALYZER) c.execute(CREATE_DOC_TABLE) c.execute(CREATE_PARTED_TABLE) c.execute(''' INSERT INTO t1 (id, text) VALUES (0, 'Phase queue is foo!') ''') insert_data(conn, 'doc', 't1', 10) c.execute(CREATE_BLOB_TABLE) run_selects(c, versions[0].version) container = conn.get_blob_container('b1') digest = container.put(BytesIO(b'sample data')) container.get(digest) self._process_on_stop() for version_def in versions[1:]: self.assert_data_persistence(version_def, nodes, digest) # restart with latest version version_def = versions[-1] self.assert_data_persistence(version_def, nodes, digest)
def _test_recovery(self, path): cluster = self._new_cluster(path.from_version, self.NUMBER_OF_NODES) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' create table doc.test(x int) clustered into 1 shards with( number_of_replicas = 1, "unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0') ''') num_docs = random.randint(0, 10) if num_docs > 0: insert_data(conn, 'doc', 'test', num_docs) if random.choice([True, False]): c.execute("refresh table doc.test") self._upgrade_to_mixed_cluster(cluster, path.to_version) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) # upgrade fully to the new version self._upgrade_cluster(cluster, path.to_version, self.NUMBER_OF_NODES) if random.choice([True, False]): c.execute("refresh table doc.test") assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test'))
def _test_retention_leases_established_when_relocating_primary(self, path): number_of_nodes = 3 cluster = self._new_cluster(path.from_version, number_of_nodes) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() number_of_shards = random.randint(1, 5) number_of_replicas = random.randint(0, 1) c.execute( '''create table doc.test(x int) clustered into ? shards with( "number_of_replicas" = ?, "soft_deletes.enabled" = false, "allocation.max_retries" = 0, "unassigned.node_left.delayed_timeout" = '100ms' )''', ( number_of_shards, number_of_replicas, )) number_of_docs = random.randint(0, 10) if number_of_docs > 0: insert_data(conn, 'doc', 'test', number_of_docs) if random.choice([True, False]): c.execute('refresh table doc.test') assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) self._upgrade_to_mixed_cluster(cluster, path.to_version) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) # trigger a primary relocation by excluding the primary from this index c.execute( '''select node['id'] from sys.shards where primary=true and table_name='test' ''' ) primary_id = c.fetchall() self.assertTrue(primary_id) c.execute( 'alter table doc.test set ("routing.allocation.exclude._id" = ?)', primary_id) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) assert_busy( lambda: self. _assert_ensure_peer_recovery_retention_leases_renewed_and_synced( conn, 'doc', 'test')) # upgrade fully to the new version self._upgrade_cluster(cluster, path.to_version, number_of_nodes) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) assert_busy( lambda: self. _assert_ensure_peer_recovery_retention_leases_renewed_and_synced( conn, 'doc', 'test'))
def test_snapshot_compatibility(self): """Test snapshot compatibility when upgrading 3.3.x -> 4.x.x Using Minio as a S3 repository, the first cluster that runs creates the repo, a table and inserts/selects some data, which then is snapshotted and deleted. The next cluster recovers the data from the last snapshot, performs further inserts/selects, to then snapshot the data and delete it. We are interested in the transition 3.3.x -> 4.x.x """ with MinioServer() as minio: t = threading.Thread(target=minio.run) t.daemon = True t.start() wait_until(lambda: _is_up('127.0.0.1', 9000)) num_nodes = 3 num_docs = 30 prev_version = None num_snapshot = 1 path_data = 'data_test_snapshot_compatibility' cluster_settings = { 'cluster.name': gen_id(), 'path.data': path_data } shutil.rmtree(path_data, ignore_errors=True) for version in self.VERSION: cluster = self._new_cluster(version, num_nodes, settings=cluster_settings) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() if not prev_version: c.execute(self.CREATE_REPOSITORY) c.execute(CREATE_ANALYZER) c.execute(CREATE_DOC_TABLE) insert_data(conn, 'doc', 't1', num_docs) else: c.execute( self.RESTORE_SNAPSHOT_TPT.format(num_snapshot - 1)) c.execute('SELECT COUNT(*) FROM t1') rowcount = c.fetchone()[0] self.assertEqual(rowcount, num_docs) run_selects(c, version) c.execute(self.CREATE_SNAPSHOT_TPT.format(num_snapshot)) c.execute(self.DROP_DOC_TABLE) self._process_on_stop() prev_version = version num_snapshot += 1 shutil.rmtree(path_data, ignore_errors=True)
def _test_recovery_with_concurrent_indexing(self, path): cluster = self._new_cluster(path.from_version, self.NUMBER_OF_NODES) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' create table doc.test(x int) clustered into 1 shards with( number_of_replicas = 2, "unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0') ''') # insert data into the initial homogeneous cluster insert_data(conn, 'doc', 'test', 10) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) # make sure that we can index while the replicas are recovering c.execute('''alter table doc.test set ("routing.allocation.enable"='primaries')''') self._upgrade_to_mixed_cluster(cluster, path.to_version) c.execute('''alter table doc.test set ("routing.allocation.enable"='all')''') # insert data into a mixed cluster insert_data(conn, 'doc', 'test', 50) c.execute('refresh table doc.test') # make sure that we can index while the replicas are recovering c.execute('select count(*) from doc.test') self.assertEqual(c.fetchone()[0], 60) # check counts for each node individually c.execute('select id from sys.nodes') node_ids = c.fetchall() self.assertEqual(len(node_ids), self.NUMBER_OF_NODES) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) for node_id in node_ids: assert_busy(lambda: self._assert_num_docs_by_node_id(conn, 'doc', 'test', node_id[0], 60)) c.execute('''alter table doc.test set ("routing.allocation.enable"='primaries')''') # upgrade the full cluster self._upgrade_cluster(cluster, path.to_version, self.NUMBER_OF_NODES) c.execute('''alter table doc.test set ("routing.allocation.enable"='all')''') insert_data(conn, 'doc', 'test', 45) c.execute('refresh table doc.test') c.execute('select count(*) from doc.test') res = c.fetchone() self.assertEqual(res[0], 105) c.execute('select id from sys.nodes') node_ids = c.fetchall() self.assertEqual(len(node_ids), self.NUMBER_OF_NODES) for node_id in node_ids: assert_busy(lambda: self._assert_num_docs_by_node_id(conn, 'doc', 'test', node_id[0], 105))
def _test_operation_based_recovery(self, path): cluster = self._new_cluster(path.from_version, self.NUMBER_OF_NODES) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' create table doc.test(x int) clustered into 1 shards with( "number_of_replicas" = 2, "soft_deletes.enabled" = true) ''') assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) insert_data(conn, 'doc', 'test', random.randint(100, 200)) c.execute('refresh table doc.test') self._assert_ensure_checkpoints_are_synced(conn, 'doc', 'test') num_docs = random.randint(0, 3) if num_docs > 0: insert_data(conn, 'doc', 'test', num_docs) self._upgrade_to_mixed_cluster(cluster, path.to_version) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) num_docs = random.randint(0, 3) if num_docs > 0: insert_data(conn, 'doc', 'test', num_docs) self._assert_ensure_checkpoints_are_synced(conn, 'doc', 'test') # upgrade fully to the new version self._upgrade_cluster(cluster, path.to_version, self.NUMBER_OF_NODES) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) num_docs = random.randint(0, 3) if num_docs > 0: insert_data(conn, 'doc', 'test', num_docs) self._assert_ensure_checkpoints_are_synced(conn, 'doc', 'test')
def _test_rolling_upgrade(self, path, nodes): """ Test a rolling upgrade across given versions. An initial test cluster is started and then subsequently each node in the cluster is upgraded to the new version. After each upgraded node a SQL statement is executed that involves all nodes in the cluster, in order to check if communication between nodes is possible. """ shards, replicas = (nodes, 1) expected_active_shards = shards + shards * replicas cluster = self._new_cluster(path.from_version, nodes) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(f''' CREATE TABLE doc.t1 ( type BYTE, value FLOAT ) CLUSTERED INTO {shards} SHARDS WITH (number_of_replicas={replicas}) ''') insert_data(conn, 'doc', 't1', 1000) c.execute(f''' CREATE TABLE doc.parted ( id INT, value INT ) CLUSTERED INTO {shards} SHARDS PARTITIONED BY (id) WITH (number_of_replicas=0, "write.wait_for_active_shards"=1) ''') c.execute("INSERT INTO doc.parted (id, value) VALUES (1, 1)") # Add the shards of the new partition primaries expected_active_shards += shards for idx, node in enumerate(cluster): new_node = self.upgrade_node(node, path.to_version) cluster[idx] = new_node with connect(new_node.http_url, error_trace=True) as conn: c = conn.cursor() wait_for_active_shards(c, expected_active_shards) c.execute(''' SELECT type, AVG(value) FROM doc.t1 GROUP BY type ''') c.fetchall() # Ensure aggregation with different intermediate input works, this was an regression for 4.1 <-> 4.2 c.execute(''' SELECT type, count(distinct value) FROM doc.t1 GROUP BY type ''') c.fetchall() # Ensure scalar symbols are working across versions c.execute(''' SELECT type, value + 1 FROM doc.t1 WHERE value > 1 LIMIT 1 ''') c.fetchone() # Ensure that inserts, which will create a new partition, are working while upgrading c.execute("INSERT INTO doc.parted (id, value) VALUES (?, ?)", [idx + 10, idx + 10]) # Add the shards of the new partition primaries expected_active_shards += shards # Finally validate that all shards (primaries and replicas) of all partitions are started # and writes into the partitioned table while upgrading were successful with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() wait_for_active_shards(c, expected_active_shards) c.execute(''' REFRESH TABLE doc.parted ''') c.execute(''' SELECT count(*) FROM doc.parted ''') res = c.fetchone() self.assertEqual(res[0], nodes + 1)
def test_snapshot_restore_and_drop_in_parallel(self): """Test to run the drop and restore operation on two different snapshots in parallel. The purpose of this test is to validate that the snapshot mechanism of CrateDB can handle the two operations in parallel. Here, Minio is used as s3 backend for the repository, but this should work on any other backend as well. """ with MinioServer() as minio: t = threading.Thread(target=minio.run) t.daemon = True t.start() wait_until(lambda: _is_up('127.0.0.1', 9000)) num_nodes = random.randint(3, 5) number_of_shards = random.randint(1, 3) number_of_replicas = random.randint(0, 2) num_docs = random.randint(1, 100) cluster_settings = { 'cluster.name': gen_id(), 'path.data': self.PATH_DATA } shutil.rmtree(self.PATH_DATA, ignore_errors=True) cluster = self._new_cluster('latest-nightly', num_nodes, settings=cluster_settings) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() wait_for_active_shards(c) c.execute( ''' create table doc.test(x int) clustered into ? shards with( number_of_replicas =?) ''', ( number_of_shards, number_of_replicas, )) insert_data(conn, 'doc', 'test', num_docs) c.execute(''' CREATE REPOSITORY repo TYPE S3 WITH (access_key = 'minio', secret_key = 'miniostorage', bucket='backups', endpoint = '127.0.0.1:9000', protocol = 'http') ''') c.execute( 'CREATE SNAPSHOT repo.snapshot1 TABLE doc.test WITH (wait_for_completion = true)' ) c.execute( 'CREATE SNAPSHOT repo.snapshot2 TABLE doc.test WITH (wait_for_completion = true)' ) c.execute('DROP TABLE doc.test') # Drop snapshot2 while the restore of snapshot1 is still running c.execute( 'RESTORE SNAPSHOT repo.snapshot1 ALL WITH (wait_for_completion = false)' ) try: c.execute('DROP SNAPSHOT repo.snapshot2') except ProgrammingError: self.fail( "Restore and Drop Snapshot operation should work in parallel" ) assert_busy(lambda: self._assert_num_docs(conn, num_docs)) cluster.stop()
def _test_relocation_with_concurrent_indexing(self, path): cluster = self._new_cluster(path.from_version, self.NUMBER_OF_NODES) cluster.start() with connect(cluster.node().http_url, error_trace=True) as conn: c = conn.cursor() c.execute(''' create table doc.test(x int) clustered into 1 shards with( "number_of_replicas" = 2, "unassigned.node_left.delayed_timeout" = '100ms', "allocation.max_retries" = '0') ''') insert_data(conn, 'doc', 'test', 10) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) # make sure that no shards are allocated, so we can make sure the primary stays # on the old node (when one node stops, we lose the master too, so a replica # will not be promoted) c.execute( '''alter table doc.test set("routing.allocation.enable"='none')''' ) self._upgrade_to_mixed_cluster(cluster, path.to_version) c.execute( '''select id from sys.nodes order by version['number'] desc limit 1''' ) new_node_id = c.fetchone()[0] c.execute( '''select id from sys.nodes order by version['number'] asc limit 1''' ) old_node_id = c.fetchone()[0] # remove the replica and guaranteed the primary is placed on the old node c.execute( '''alter table doc.test set ( "number_of_replicas"=0, "routing.allocation.enable"='all', "routing.allocation.include._id"=? )''', (old_node_id, )) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) c.execute( '''alter table doc.test set ("routing.allocation.include._id"=?)''', (new_node_id, )) insert_data(conn, 'doc', 'test', 50) # ensure the relocation from old node to new node has occurred; otherwise the table is green # even though shards haven't moved to the new node yet (allocation was throttled). assert_busy(lambda: self._assert_shard_state( conn, 'doc', 'test', new_node_id, 'STARTED')) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) c.execute('refresh table doc.test') self._assert_num_docs_by_node_id(conn, 'doc', 'test', new_node_id, 60) # upgrade fully to the new version self._upgrade_cluster(cluster, path.to_version, self.NUMBER_OF_NODES) c.execute('''alter table doc.test set("number_of_replicas"=2)''') c.execute( '''alter table doc.test reset("routing.allocation.include._id")''' ) insert_data(conn, 'doc', 'test', 45) assert_busy(lambda: self._assert_is_green(conn, 'doc', 'test')) c.execute('refresh table doc.test') c.execute('select id from sys.nodes') node_ids = c.fetchall() self.assertEqual(len(node_ids), self.NUMBER_OF_NODES) for node_id in node_ids: self._assert_num_docs_by_node_id(conn, 'doc', 'test', node_id[0], 105)