def test_delete_and_read(num_reads):
from binlog.reader import TDSReader
from binlog.writer import TDSWriter
MAX_LOG_EVENTS = 10
try:
tmpdir = mktemp()
writer = TDSWriter(tmpdir, max_log_events=MAX_LOG_EVENTS)
reader = TDSReader(tmpdir, checkpoint='test')
for x in range(25):
writer.append(str(x).encode("ascii"))
for x in range(num_reads):
reader.next_record()
if num_reads > MAX_LOG_EVENTS:
writer.delete(1)
else:
with pytest.raises(ValueError):
writer.delete(1)
pass
except:
raise
finally:
shutil.rmtree(tmpdir)
def test_Server_protocol_not_store_in_binlog_when_no_data():
from binlog.server import Server
from binlog.reader import TDSReader
import tempfile
with tempfile.TemporaryDirectory() as base:
s = Server(base, "/tmp/server.sock")
p = s.get_protocol()()
p.connection_made(None)
p.connection_lost(None)
r = TDSReader(base)
assert r.next_record() is None
def test_Server_protocol_not_store_in_binlog_when_no_data():
from binlog.server import Server
from binlog.reader import TDSReader
import tempfile
with tempfile.TemporaryDirectory() as base:
s = Server(base, "/tmp/server.sock")
p = s.get_protocol()()
p.connection_made(None)
p.connection_lost(None)
r = TDSReader(base)
assert r.next_record() is None
def test_status_and_delete_match(reads):
"""reader.status is consistent with writer.delete."""
from binlog.reader import TDSReader
from binlog.writer import TDSWriter
try:
tmpdir = mktemp()
writer = TDSWriter(tmpdir, max_log_events=MAX_LOG_EVENTS)
reader = TDSReader(tmpdir, checkpoint='test')
for x in range(25):
writer.append(str(x).encode("ascii"))
for x in range(reads):
rec = reader.next_record()
reader.ack(rec)
reader.save()
status = reader.status()
for idx, can_delete in status.items():
if idx == max(status.keys()):
# We cannot delete the last DB cause is current.
assert not can_delete
with pytest.raises(ValueError):
writer.delete(idx)
else:
if can_delete:
writer.delete(idx)
except:
raise
finally:
shutil.rmtree(tmpdir)
def test_Server_protocol_store_in_binlog():
from binlog.server import Server
from binlog.reader import TDSReader
import tempfile
with tempfile.TemporaryDirectory() as base:
s = Server(base, "/tmp/server.sock")
p = s.get_protocol()()
p.connection_made(None)
p.data_received(b"TEST")
p.connection_lost(None)
r = TDSReader(base)
assert r.next_record().value == b"TEST"
def test_Server_protocol_store_in_binlog():
from binlog.server import Server
from binlog.reader import TDSReader
import tempfile
with tempfile.TemporaryDirectory() as base:
s = Server(base, "/tmp/server.sock")
p = s.get_protocol()()
p.connection_made(None)
p.data_received(b"TEST")
p.connection_lost(None)
r = TDSReader(base)
assert r.next_record().value == b"TEST"
def test_status_and_delete_match(reads):
"""reader.status is consistent with writer.delete."""
from binlog.reader import TDSReader
from binlog.writer import TDSWriter
try:
tmpdir = mktemp()
writer = TDSWriter(tmpdir, max_log_events=MAX_LOG_EVENTS)
reader = TDSReader(tmpdir, checkpoint='test')
for x in range(25):
writer.append(x)
for x in range(reads):
rec = reader.next_record()
reader.ack(rec)
reader.save()
status = reader.status()
for idx, can_delete in status.items():
if idx == max(status.keys()):
# We cannot delete the last DB cause is current.
assert not can_delete
with pytest.raises(ValueError):
writer.delete(idx)
else:
if can_delete:
writer.delete(idx)
except:
raise
finally:
shutil.rmtree(tmpdir)
def test_status_method_works_after_deletion():
from binlog.reader import TDSReader
from binlog.writer import TDSWriter
try:
tmpdir = mktemp()
writer = TDSWriter(tmpdir, max_log_events=10)
reader = TDSReader(tmpdir, checkpoint='test')
for x in range(25):
writer.append(x)
reader.status()
writer.delete(1)
reader.status()
except:
raise
finally:
shutil.rmtree(tmpdir)
def test_reader_starts_with_the_lowest_database_available():
from binlog.reader import TDSReader
from binlog.writer import TDSWriter
try:
tmpdir = mktemp()
writer = TDSWriter(tmpdir, max_log_events=10)
for x in range(25):
writer.append(str(x).encode("ascii"))
writer.delete(1)
reader = TDSReader(tmpdir, checkpoint='test')
for x in range(10, 25):
rec = reader.next_record()
assert int(rec.value) == x
assert reader.next_record() is None
except:
raise
finally:
shutil.rmtree(tmpdir)
def test_Server_concurrent_writes(server_factory, data):
with server_factory() as server:
socket_path, env_path = server
def storedatum(d):
s = socket.socket(socket.AF_UNIX)
s.connect(socket_path)
s.sendall(d)
s.close()
with ThreadPoolExecutor(max_workers=64) as pool:
for res in pool.map(storedatum, data):
assert res is None
expected_set = collections.Counter(data)
reader = TDSReader(env_path)
current_set = collections.Counter(d.value
for d in iter(reader.next_record, None))
assert current_set == expected_set
def test_status_method_works_after_deletion():
from binlog.reader import TDSReader
from binlog.writer import TDSWriter
try:
tmpdir = mktemp()
writer = TDSWriter(tmpdir, max_log_events=10)
reader = TDSReader(tmpdir, checkpoint='test')
for x in range(25):
writer.append(str(x).encode("ascii"))
reader.status()
writer.delete(1)
reader.status()
except:
raise
finally:
shutil.rmtree(tmpdir)
from itertools import cycle
import atexit
import os
import signal
import time
from binlog.reader import TDSReader
r = TDSReader('test', checkpoint='status')
def save_status():
"""
Save a checkpoint with the current status.
Only the acknowledged entries will be included in the checkpoint.
"""
r.save()
atexit.register(save_status)
for i in cycle(range(1, 20001)):
if i == 20000: # Make a checkpoint each 20k reads.
save_status()
n = r.next_record() # Read the next log entry.
if n is None:
time.sleep(1)
else: