def test_wal_removal(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 2 zenith_env_builder.broker = True # to advance remote_consistent_llsn zenith_env_builder.enable_local_fs_remote_storage() env = zenith_env_builder.init_start() env.zenith_cli.create_branch('test_safekeepers_wal_removal') pg = env.postgres.create_start('test_safekeepers_wal_removal') with closing(pg.connect()) as conn: with conn.cursor() as cur: # we rely upon autocommit after each statement # as waiting for acceptors happens there cur.execute('CREATE TABLE t(key int primary key, value text)') cur.execute( "INSERT INTO t SELECT generate_series(1,100000), 'payload'") tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0] timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0] # force checkpoint to advance remote_consistent_lsn with closing(env.pageserver.connect()) as psconn: with psconn.cursor() as pscur: pscur.execute(f"checkpoint {tenant_id} {timeline_id}") # We will wait for first segment removal. Make sure they exist for starter. first_segments = [ os.path.join(sk.data_dir(), tenant_id, timeline_id, '000000010000000000000001') for sk in env.safekeepers ] assert all(os.path.exists(p) for p in first_segments) http_cli = env.safekeepers[0].http_client() # Pretend WAL is offloaded to s3. http_cli.record_safekeeper_info(tenant_id, timeline_id, {'s3_wal_lsn': 'FFFFFFFF/FEFFFFFF'}) # wait till first segment is removed on all safekeepers started_at = time.time() while True: if all(not os.path.exists(p) for p in first_segments): break elapsed = time.time() - started_at if elapsed > 20: raise RuntimeError( f"timed out waiting {elapsed:.0f}s for first segment get removed" ) time.sleep(0.5)
def test_normal_work(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 zenith_env_builder.broker = True env = zenith_env_builder.init_start() env.zenith_cli.create_branch('test_safekeepers_normal_work') pg = env.postgres.create_start('test_safekeepers_normal_work') with closing(pg.connect()) as conn: with conn.cursor() as cur: # we rely upon autocommit after each statement # as waiting for acceptors happens there cur.execute('CREATE TABLE t(key int primary key, value text)') cur.execute( "INSERT INTO t SELECT generate_series(1,100000), 'payload'") cur.execute('SELECT sum(key) FROM t') assert cur.fetchone() == (5000050000, )
def test_broker(zenith_env_builder: ZenithEnvBuilder): zenith_env_builder.num_safekeepers = 3 zenith_env_builder.broker = True zenith_env_builder.enable_local_fs_remote_storage() env = zenith_env_builder.init_start() env.zenith_cli.create_branch("test_broker", "main") pg = env.postgres.create_start('test_broker') pg.safe_psql("CREATE TABLE t(key int primary key, value text)") # learn zenith timeline from compute tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0] timeline_id = pg.safe_psql("show zenith.zenith_timeline")[0][0] # wait until remote_consistent_lsn gets advanced on all safekeepers clients = [sk.http_client() for sk in env.safekeepers] stat_before = [ cli.timeline_status(tenant_id, timeline_id) for cli in clients ] log.info(f"statuses is {stat_before}") pg.safe_psql("INSERT INTO t SELECT generate_series(1,100), 'payload'") # force checkpoint to advance remote_consistent_lsn with closing(env.pageserver.connect()) as psconn: with psconn.cursor() as pscur: pscur.execute(f"checkpoint {tenant_id} {timeline_id}") # and wait till remote_consistent_lsn propagates to all safekeepers started_at = time.time() while True: stat_after = [ cli.timeline_status(tenant_id, timeline_id) for cli in clients ] if all( lsn_from_hex(s_after.remote_consistent_lsn) > lsn_from_hex( s_before.remote_consistent_lsn) for s_after, s_before in zip(stat_after, stat_before)): break elapsed = time.time() - started_at if elapsed > 20: raise RuntimeError( f"timed out waiting {elapsed:.0f}s for remote_consistent_lsn propagation: status before {stat_before}, status current {stat_after}" ) time.sleep(0.5)