def test_select_from_timeout(self):
from relstorage.tests import mock
from relstorage.adapters.interfaces import AggregateOperationTimeoutError
cursor = MockCursor()
cursor.sort_sequence_params = True
cursor.many_results = [[(1, 1)], [(2, 1)], [(3, 1)], []]
batcher = self.getClass()(cursor)
batcher.bind_limit = 1
batcher.perf_counter = mock.Mock()
# These will be the time values returned from perf_counter()
batcher.perf_counter.side_effect = (
12345, # Begin
12346, # First batch
12347, # Second batch
)
gener = batcher.select_from((
'zoid',
'tid',
),
'object_state',
timeout=2,
oids=[1, 2, 3, 4, 5])
rows = []
with self.assertRaises(AggregateOperationTimeoutError):
for row in gener:
rows.append(row)
# We ran exactly twice before the perf_counter exceeded the timeout.
self.assertEqual(rows, [
(1, 1),
(2, 1),
])
def test_select_multiple_many_batch(self):
cursor = MockCursor()
cursor.many_results = [
[(1, 1)],
[(3, 1)],
[]
]
batcher = self.getClass()(cursor)
batcher.row_limit = 2
rows = batcher.select_from(('zoid', 'tid'), 'object_state',
oids=(1, 2, 3, 4, 5))
rows = list(rows)
self.assertEqual(cursor.executed, [
('SELECT zoid,tid FROM object_state WHERE oids = ANY (%s)',
([1, 2,],)),
('SELECT zoid,tid FROM object_state WHERE oids = ANY (%s)',
([3, 4,],)),
('SELECT zoid,tid FROM object_state WHERE oids = ANY (%s)',
([5,],)),
])
self.assertEqual(rows, [
(1, 1),
(3, 1)
])
def test_get_tid_empty_db(self):
inst = self._makeOne()
inst.poller.poll_tid = 0
cur = MockCursor()
cur.results = None
self.assertEqual(inst.get_tid(cur), 0)
def test_without_replica_conf(self):
cm = self._makeOne()
conn = MockConnection()
cm.restart_load(conn, MockCursor())
self.assertTrue(conn.rolled_back)
conn = MockConnection()
cm.restart_store(conn, MockCursor())
self.assertTrue(conn.rolled_back)
def test_select_multiple_one_batch(self):
cursor = MockCursor()
cursor.sort_sequence_params = True
batcher = self.getClass()(cursor)
list(
batcher.select_from(('zoid', 'tid'),
'object_state',
oids=(1, 2, 3, 4)))
self.assertEqual(
cursor.executed,
[(self.select_multiple_one_batch, self._in(1, 2, 3, 4))])
def test_delete_auto_flush(self):
cursor = MockCursor()
cursor.sort_sequence_params = True
batcher = self.getClass()(cursor, 2)
batcher.sorted_deletes = True
batcher.delete_from("mytable", id=2)
batcher.delete_from("mytable", id=1)
self.assertEqual(cursor.executed,
[(self.delete_auto_flush, self._in(1, 2))])
self.assertEqual(batcher.rows_added, 0)
self.assertEqual(batcher.size_added, 0)
self.assertEqual(batcher.deletes, {})
self.assertEqual(batcher.total_rows_inserted, 0)
self.assertEqual(batcher.total_rows_deleted, 2)
self.assertEqual(batcher.total_size_inserted, 0)
def test_insert_two_raw_rows(self):
class MockRawType(object):
pass
cursor = MockCursor()
batcher = self.getClass()(cursor, {'rawdata': MockRawType})
batcher.insert_into(
"mytable (id, data)",
":id, :rawdata",
{'id': 1, 'rawdata': 'xyz'},
rowkey=1,
size=3,
)
batcher.insert_into(
"mytable (id, data)",
":id, :rawdata",
{'id': 2, 'rawdata': 'abc'},
rowkey=2,
size=3,
)
batcher.flush()
self.assertEqual(
cursor.executed,
[(
'INSERT ALL\n'
'INTO mytable (id, data) VALUES (:id_0, :rawdata_0)\n'
'INTO mytable (id, data) VALUES (:id_1, :rawdata_1)\n'
'SELECT * FROM DUAL',
{'id_0': 1, 'id_1': 2, 'rawdata_0': 'xyz', 'rawdata_1': 'abc'})
])
self.assertEqual(cursor.inputsizes, {
'rawdata_0': MockRawType,
'rawdata_1': MockRawType,
})
def test_insert_two_rows(self):
cursor = MockCursor()
batcher = self.getClass()(cursor, {})
batcher.insert_into(
"mytable (id, name)",
":id, :id || :name",
{'id': 1, 'name': 'a'},
rowkey=1,
size=3,
)
batcher.insert_into(
"mytable (id, name)",
":id, :id || :name",
{'id': 2, 'name': 'b'},
rowkey=2,
size=3,
)
self.assertEqual(cursor.executed, [])
batcher.flush()
self.assertEqual(
cursor.executed,
[(
'INSERT ALL\n'
'INTO mytable (id, name) VALUES (:id_0, :id_0 || :name_0)\n'
'INTO mytable (id, name) VALUES (:id_1, :id_1 || :name_1)\n'
'SELECT * FROM DUAL',
{'id_0': 1, 'id_1': 2, 'name_1': 'b', 'name_0': 'a'})
])
def test_query(self):
inst = self._makeOne()
unbound = type(inst)._iter_objects_query
self.assertEqual(
str(unbound),
'SELECT zoid, state FROM object_state WHERE (tid = %(tid)s) ORDER BY zoid'
)
# Bound we get pyformat %(name)s replaced with :name
self.assertEqual(
str(inst._iter_objects_query),
'SELECT zoid, state FROM object_state WHERE (tid = :tid) ORDER BY zoid'
)
cursor = MockCursor()
list(inst.iter_objects(cursor, 1)) # Iterator, must flatten
self.assertEqual(1, len(cursor.executed))
stmt, params = cursor.executed[0]
self.assertEqual(
stmt,
'SELECT zoid, state FROM object_state WHERE (tid = :tid) ORDER BY zoid'
)
self.assertEqual(params, {'tid': 1})
def test_insert_defer_multi_table(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'a'),
rowkey=1,
size=3,
)
batcher.insert_into(
"othertable (name)",
"?",
('a'),
rowkey=1,
size=1,
)
self.assertEqual(cursor.executed, [])
self.assertEqual(batcher.rows_added, 2)
self.assertEqual(batcher.size_added, 4)
self.assertEqual(dict(batcher.inserts), {
('INSERT', 'mytable (id, name)', '%s, id || %s', ''): {1: (1, 'a')},
('INSERT', 'othertable (name)', '?', ''): {1: ('a')},
})
self.assertEqual(batcher.total_rows_inserted, 0)
self.assertEqual(batcher.total_rows_deleted, 0)
self.assertEqual(batcher.total_size_inserted, 0)
def test_insert_auto_flush_multi_table(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.size_limit = 10
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'a'),
rowkey=1,
size=5,
)
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(2, 'B'),
rowkey=2,
size=5,
)
self.assertEqual(
cursor.executed,
[(
'INSERT INTO mytable (id, name) VALUES\n'
'(%s, id || %s),\n'
'(%s, id || %s)\n',
(1, 'a', 2, 'B'))
])
self.assertEqual(batcher.rows_added, 0)
self.assertEqual(batcher.size_added, 0)
self.assertEqual(batcher.inserts, {})
self.assertEqual(batcher.total_rows_inserted, 2)
self.assertEqual(batcher.total_rows_deleted, 0)
self.assertEqual(batcher.total_size_inserted, 10)
def test_insert_duplicate(self):
# A second insert on the same rowkey replaces the first insert.
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'a'),
rowkey=1,
size=3,
)
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'b'),
rowkey=1,
size=3,
)
self.assertEqual(cursor.executed, [])
self.assertEqual(batcher.rows_added, 2)
self.assertEqual(batcher.size_added, 6)
self.assertEqual(batcher.inserts, {
('INSERT', 'mytable (id, name)', '%s, id || %s', ''): {
1: (1, 'b')
}
})
def test_select_one(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
list(batcher.select_from(('zoid', 'tid'), 'object_state', oids=(1,)))
self.assertEqual(cursor.executed, [
('SELECT zoid,tid FROM object_state WHERE oids = ANY (%s)',
([1,],))
])
def test_delete_multiple_column(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.delete_from("mytable", id=2, tid=10)
self.assertEqual(cursor.executed, [])
self.assertEqual(batcher.rows_added, 1)
self.assertEqual(batcher.size_added, 0)
self.assertEqual(dict(batcher.deletes),
{('mytable', ('id', 'tid')): set([(2, 10)])})
def test_select_multiple_many_batch(self, batch_limit_attr='row_limit'):
cursor = MockCursor()
cursor.sort_sequence_params = True
cursor.many_results = [[(1, 1)], [(3, 1)], []]
batcher = self.getClass()(cursor)
setattr(batcher, batch_limit_attr, 2)
rows = batcher.select_from(('zoid', 'tid'),
'object_state',
oids=iter((1, 2, 3, 4, 5)))
rows = list(rows)
self.assertEqual(cursor.executed, [
(self.select_multiple_many_batch, self._in(1, 2)),
(self.select_multiple_many_batch, self._in(3, 4)),
(self.select_one, self._in(5)),
])
self.assertEqual(rows, [(1, 1), (3, 1)])
def test_select_multiple_one_batch(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
list(batcher.select_from(('zoid', 'tid'), 'object_state',
oids=(1, 2, 3, 4)))
self.assertEqual(cursor.executed, [
('SELECT zoid,tid FROM object_state WHERE oids IN (%s,%s,%s,%s)',
(1, 2, 3, 4))
])
def test_rollback_quietly_reports_exception(self):
cm = self._makeOne()
meth = cm.rollback_quietly
# With no exceptions, we should get a success report.
self.assertTrue(meth(MockConnection(), MockCursor()))
class O(object):
closed = False
fetched = False
ex = CloseException
def rollback(self):
raise self.ex
def close(self):
self.closed = True
raise CloseException
def fetchall(self):
self.fetched = True
o = O()
# As a cursor, we fetchall, but don't close, so we get no report.
self.assertTrue(meth(MockConnection(), o))
self.assertTrue(o.fetched)
self.assertFalse(o.closed)
o = O()
# As a connection the error is reported and everything is closed.
self.assertFalse(meth(o, MockCursor()))
self.assertTrue(o.closed)
# Raising DisconnectedException instead of CloseException
o = O()
o.ex = DisconnectedException
self.assertTrue(meth(MockConnection(), o))
self.assertTrue(o.fetched)
self.assertFalse(o.closed)
o = O()
o.ex = DisconnectedException
self.assertFalse(meth(o, MockCursor()))
self.assertTrue(o.closed)
def test_with_ro_replica_conf(self):
import os
import relstorage.tests
tests_dir = relstorage.tests.__file__
replica_conf = os.path.join(os.path.dirname(tests_dir),
'replicas.conf')
ro_replica_conf = os.path.join(os.path.dirname(tests_dir),
'ro_replicas.conf')
cm = self._makeOne(replica_conf=replica_conf,
ro_replica_conf=ro_replica_conf)
conn = MockConnection()
conn.replica = 'readonlyhost'
cm.restart_load(conn, MockCursor())
self.assertTrue(conn.rolled_back)
conn.replica = 'other'
self.assertRaises(interfaces.ReplicaClosedException, cm.restart_load,
conn, MockCursor())
def test_delete_defer(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.delete_from("mytable", id=2)
self.assertEqual(cursor.executed, [])
self.assertEqual(batcher.rows_added, 1)
self.assertEqual(batcher.size_added, 0)
self.assertEqual(batcher.total_rows_inserted, 0)
self.assertEqual(batcher.total_rows_deleted, 0)
self.assertEqual(batcher.total_size_inserted, 0)
self.assertEqual(dict(batcher.deletes),
{('mytable', ('id',)): set([(2,)])})
def test_update_set_static(self):
cursor = MockCursor()
batcher = self.getClass()(cursor, 2)
cnt = batcher.update_set_static('UPDATE pack_object SET foo=1',
zoid=iter((1, 2, 3, 4, 5, 6, 7)))
self.assertEqual(cnt, 7)
self.assertEqual(cursor.executed, [
(self.update_set_static_stmt, self._in(2, 1, do_sort=False)),
(self.update_set_static_stmt, self._in(4, 3, do_sort=False)),
(self.update_set_static_stmt, self._in(6, 5, do_sort=False)),
(self.update_set_static_stmt.replace(',%s', ''), self._in(7)),
])
def test_8(self):
mvd = MVD()
cursor = MockCursor()
cursor.results.append((b'8.0.19-standard',))
vi = mvd.get_version_info(cursor)
self.assertEqual(vi, (8, 0, 19))
self.assertEqual(mvd.get_major_version(cursor), 8)
self.assertTrue(mvd.supports_nowait(None))
self.assertTrue(mvd.supports_transaction_isolation(None))
self.assertEqual(mvd.get_version(cursor), '8.0.19-standard')
self.assertGreater(mvd.get_version_info(cursor), (5, 7, 20))
self.assertEqual([('SELECT version()', None)], cursor.executed)
def test_add_transaction_hp(self):
inst = self._makeOne()
cur = MockCursor(self)
__traceback_info__ = inst.__dict__
inst.add_transaction(cur, 1, u'user', u'desc', u'ext')
self.assertEqual(cur.executed.pop(),
(str(inst._add_transaction_query),
(1, False, b'user', b'desc', b'ext')))
inst.add_transaction(cur, 1, u'user', u'desc', u'ext', packed=True)
self.assertEqual(cur.executed.pop(),
(str(inst._add_transaction_query),
(1, True, b'user', b'desc', b'ext')))
def test_delete_auto_flush(self):
cursor = MockCursor()
batcher = self.getClass()(cursor, 2)
batcher.sorted_deletes = True
batcher.delete_from("mytable", id=2)
batcher.delete_from("mytable", id=1)
self.assertEqual(cursor.executed,
[('DELETE FROM mytable WHERE id IN (%s,%s)', ((1, 2)))])
self.assertEqual(batcher.rows_added, 0)
self.assertEqual(batcher.size_added, 0)
self.assertEqual(batcher.deletes, {})
self.assertEqual(batcher.total_rows_inserted, 0)
self.assertEqual(batcher.total_rows_deleted, 2)
self.assertEqual(batcher.total_size_inserted, 0)
def test_5(self):
mvd = MVD()
cursor = MockCursor()
cursor.results.append((b'5.7.27-log',))
vi = mvd.get_version_info(cursor)
self.assertEqual(vi, (5, 7, 27))
self.assertEqual(mvd.get_major_version(cursor), 5)
self.assertFalse(mvd.supports_nowait(None))
self.assertEqual(mvd.get_version(cursor), '5.7.27-log')
self.assertGreater(mvd.get_version_info(cursor), (5, 7, 26))
self.assertGreater(mvd.get_version_info(cursor), (5, 7, 20))
self.assertTrue(mvd.supports_good_stored_procs(None))
self.assertEqual([('SELECT version()', None)], cursor.executed)
def test_rollback_raises_exception(self):
cm = self._makeOne()
# With no exceptions, we should get a success report.
self.assertTrue(cm.rollback(MockConnection(), MockCursor()))
class O(object):
closed = False
fetched = False
ex = CloseException
def rollback(self):
raise self.ex
def close(self):
self.closed = True
raise CloseException
def fetchall(self):
self.fetched = True
o = O()
# As a cursor, there's only fetchall(), so nothing is reported.
conn = MockConnection()
self.assertTrue(cm.rollback(conn, o))
self.assertTrue(o.fetched)
self.assertFalse(o.closed)
self.assertFalse(conn.closed)
# As a connection the error is reraised and everything is closed.
o = O()
cur = MockCursor()
cur.fetchall = lambda: setattr(cur, 'fetched', True)
with self.assertRaises(o.ex):
cm.rollback(o, cur)
self.assertTrue(o.closed)
self.assertTrue(cur.fetched) # pylint:disable=no-member
self.assertTrue(cur.closed)
# Raising DisconnectedException instead of CloseException
o = O()
o.ex = DisconnectedException
conn = MockConnection()
self.assertTrue(cm.rollback(conn, o))
self.assertTrue(o.fetched)
self.assertFalse(o.closed)
self.assertFalse(conn.closed)
o = O()
o.ex = DisconnectedException
cur = MockCursor()
with self.assertRaises(o.ex):
cm.rollback(o, cur)
self.assertTrue(o.closed)
self.assertTrue(cur.closed)
def test_insert_one_row(self):
cursor = MockCursor()
batcher = self.getClass()(cursor, {})
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'a'),
rowkey=1,
size=3,
)
self.assertEqual(cursor.executed, [])
batcher.flush()
self.assertEqual(cursor.executed, [
('INSERT INTO mytable (id, name) VALUES (%s, id || %s)', (1, 'a')),
])
def test_iter_transactions(self):
inst = self._makeOne()
cursor = MockCursor()
inst.iter_transactions(cursor)
self.assertEqual(1, len(cursor.executed))
stmt, params = cursor.executed[0]
self.assertEqual(
stmt, 'SELECT tid, username, description, extension '
'FROM transaction '
"WHERE ((packed = 'N' AND tid <> :literal_0)) "
'ORDER BY tid DESC')
self.assertEqual(params, {'literal_0': 0})
def test_insert_replace(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'a'),
rowkey=1,
size=3,
command='REPLACE',
)
self.assertEqual(cursor.executed, [])
self.assertEqual(batcher.rows_added, 1)
self.assertEqual(batcher.size_added, 3)
self.assertEqual(batcher.inserts, {
('REPLACE', 'mytable (id, name)', '%s, id || %s', ''): {1: (1, 'a')}
})
def test_insert_one_raw_row(self):
class MockRawType(object):
pass
cursor = MockCursor()
batcher = self.getClass()(cursor, {'rawdata': MockRawType})
batcher.insert_into(
"mytable (id, data)",
":id, :rawdata",
{'id': 1, 'rawdata': 'xyz'},
rowkey=1,
size=3,
)
batcher.flush()
self.assertEqual(cursor.executed, [
('INSERT INTO mytable (id, data) VALUES (:id, :rawdata)',
{'id': 1, 'rawdata': 'xyz'})
])
self.assertEqual(cursor.inputsizes, {'rawdata': MockRawType})
def test_insert_defer(self):
cursor = MockCursor()
batcher = self.getClass()(cursor)
batcher.insert_into(
"mytable (id, name)",
"%s, id || %s",
(1, 'a'),
rowkey=1,
size=3,
)
self.assertEqual(cursor.executed, [])
self.assertEqual(batcher.rows_added, 1)
self.assertEqual(batcher.size_added, 3)
self.assertEqual(batcher.inserts, {
('INSERT', 'mytable (id, name)', '%s, id || %s', ''): {1: (1, 'a')}
})
self.assertEqual(batcher.total_rows_inserted, 0)
self.assertEqual(batcher.total_rows_deleted, 0)
self.assertEqual(batcher.total_size_inserted, 0)
