def test_datetime_get_set_from_database():
datetime_str = "1977-05-04 12:34:56.78"
datetime_uni = ustr(datetime_str)
datetime_obj = datetime(1977, 5, 4, 12, 34, 56, 780000)
variable = DateTimeVariable()
variable.set(datetime_str, from_db=True)
assert variable.get() == datetime_obj
variable.set(datetime_uni, from_db=True)
assert variable.get() == datetime_obj
variable.set(datetime_obj, from_db=True)
assert variable.get() == datetime_obj
datetime_str = "1977-05-04 12:34:56"
datetime_uni = ustr(datetime_str)
datetime_obj = datetime(1977, 5, 4, 12, 34, 56)
variable.set(datetime_str, from_db=True)
assert variable.get() == datetime_obj
variable.set(datetime_uni, from_db=True)
assert variable.get() == datetime_obj
variable.set(datetime_obj, from_db=True)
assert variable.get() == datetime_obj
with pytest.raises(TypeError):
variable.set(0, from_db=True)
with pytest.raises(TypeError):
variable.set(marker, from_db=True)
with pytest.raises(ValueError):
variable.set("foobar", from_db=True)
with pytest.raises(ValueError):
variable.set("foo bar", from_db=True)
def test_time_get_set_from_database():
time_str = "12:34:56.78"
time_uni = ustr(time_str)
time_obj = time(12, 34, 56, 780000)
variable = TimeVariable()
variable.set(time_str, from_db=True)
assert variable.get() == time_obj
variable.set(time_uni, from_db=True)
assert variable.get() == time_obj
variable.set(time_obj, from_db=True)
assert variable.get() == time_obj
time_str = "12:34:56"
time_uni = ustr(time_str)
time_obj = time(12, 34, 56)
variable.set(time_str, from_db=True)
assert variable.get() == time_obj
variable.set(time_uni, from_db=True)
assert variable.get() == time_obj
variable.set(time_obj, from_db=True)
assert variable.get() == time_obj
with pytest.raises(TypeError):
variable.set(0, from_db=True)
with pytest.raises(TypeError):
variable.set(marker, from_db=True)
with pytest.raises(ValueError):
variable.set("foobar", from_db=True)
def test_concurrent_behavior(self):
"""The default behavior should be to handle transactions in isolation.
Data committed in one transaction shouldn't be visible to another
running transaction before the later is committed or aborted. If
this isn't the case, the caching made by Storm (or by anything
that works with data in memory, in fact) becomes a dangerous thing.
For PostgreSQL, isolation level must be SERIALIZABLE.
For SQLite, the isolation level already is SERIALIZABLE when not
in autocommit mode. OTOH, PySQLite is nuts regarding transactional
behavior, and will easily offer READ COMMITTED behavior inside a
"transaction" (it didn't tell SQLite to open a transaction, in fact).
"""
connection1 = self.connection
connection2 = self.database.connect()
try:
result = connection1.execute("SELECT title FROM test WHERE id=10")
assert result.get_one() == ("Title 10",)
try:
connection2.execute("UPDATE test SET title='Title 100' "
"WHERE id=10")
connection2.commit()
except OperationalError as e:
assert "locked" in ustr(e) # SQLite blocks
result = connection1.execute("SELECT title FROM test WHERE id=10")
assert result.get_one() == ("Title 10",)
finally:
connection1.rollback()
def test_commit_timeout(self):
"""Regression test for commit observing the timeout.
In 0.10, the timeout wasn't observed for connection.commit().
"""
# Create a database with a table.
database = create_database("sqlite:%s?timeout=0.3" % self.get_path())
connection1 = database.connect()
connection1.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)")
connection1.commit()
# Put some data in, but also make a second connection to the database,
# which will prevent a commit until it is closed.
connection1.execute("INSERT INTO test VALUES (1)")
connection2 = database.connect()
connection2.execute("SELECT id FROM test")
started = time.time()
try:
connection1.commit()
except OperationalError as exception:
assert ustr(exception) == "database is locked"
# In 0.10, the next assertion failed because the timeout wasn't
# enforced for the "COMMIT" statement.
assert time.time() - started >= 0.3
else:
self.fail("OperationalError not raised")
def test_set_none_with_allow_none_and_column():
column = Column("column_name")
variable = CustomVariable(allow_none=False, column=column)
try:
variable.set(None)
assert False
except NoneError as e:
assert "column_name" in ustr(e)
def set_variable(variable, value):
if isinstance(variable, RawStrVariable):
# On Python 3 sqlite will return unicode objects when possible
try:
if not isinstance(value, bstr):
value = ustr(value).encode("utf-8")
except:
value = bstr(value)
variable.set(value, from_db=True)
def test_timedelta_get_set_from_database():
delta_str = "42 days 12:34:56.78"
delta_uni = ustr(delta_str)
delta_obj = timedelta(days=42,
hours=12,
minutes=34,
seconds=56,
microseconds=780000)
variable = TimeDeltaVariable()
variable.set(delta_str, from_db=True)
assert variable.get() == delta_obj
variable.set(delta_uni, from_db=True)
assert variable.get() == delta_obj
variable.set(delta_obj, from_db=True)
assert variable.get() == delta_obj
delta_str = "1 day, 12:34:56"
delta_uni = ustr(delta_str)
delta_obj = timedelta(days=1, hours=12, minutes=34, seconds=56)
variable.set(delta_str, from_db=True)
assert variable.get() == delta_obj
variable.set(delta_uni, from_db=True)
assert variable.get() == delta_obj
variable.set(delta_obj, from_db=True)
assert variable.get() == delta_obj
with pytest.raises(TypeError):
variable.set(0, from_db=True)
with pytest.raises(TypeError):
variable.set(marker, from_db=True)
with pytest.raises(ValueError):
variable.set("foobar", from_db=True)
# Intervals of months or years can not be converted to a
# Python timedelta, so a ValueError exception is raised:
with pytest.raises(ValueError):
variable.set("42 months", from_db=True)
with pytest.raises(ValueError):
variable.set("42 years", from_db=True)
def test_tracing_check_disconnect(self):
tracer = FakeTracer()
tracer_mock = self.mocker.patch(tracer)
tracer_mock.connection_raw_execute(ARGS)
self.mocker.throw(DatabaseError('connection closed'))
self.mocker.replay()
install_tracer(tracer_mock)
self.connection.is_disconnection_error = (
lambda exc, extra_disconnection_errors=():
"connection closed" in ustr(exc))
with pytest.raises(DisconnectionError):
self.connection.execute("something")
def to_database(self, params):
"""
Like L{Connection.to_database}, but this converts datetime types
to strings and byte strings to L{psycopg2.Binary} instances.
"""
for param in params:
if isinstance(param, Variable):
param = param.get(to_db=True)
if isinstance(param, (datetime, date, time, timedelta)):
yield ustr(param)
elif isinstance(param, bstr):
yield psycopg2.Binary(param)
else:
yield param
def _check_disconnect(self, function, *args, **kwargs):
"""Run the given function, checking for database disconnections."""
# Allow the caller to specify additional exception types that
# should be treated as possible disconnection errors.
extra_disconnection_errors = kwargs.pop('extra_disconnection_errors',
())
try:
return function(*args, **kwargs)
except Exception as exc:
if self.is_disconnection_error(exc, extra_disconnection_errors):
self._state = STATE_DISCONNECTED
self._raw_connection = None
raise DisconnectionError(ustr(exc))
else:
raise
def to_database(params):
"""
Like L{Connection.to_database}, but this also converts
instances of L{datetime} types to strings, and strings
instances to C{buffer} instances.
"""
for param in params:
if isinstance(param, Variable):
param = param.get(to_db=True)
if isinstance(param, (datetime, date, time, timedelta)):
yield ustr(param)
elif isinstance(param, bstr):
yield buffer(param)
else:
yield param
def test_timeout(self):
database = create_database("sqlite:%s?timeout=0.3" % self.get_path())
connection1 = database.connect()
connection2 = database.connect()
connection1.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)")
connection1.commit()
connection1.execute("INSERT INTO test VALUES (1)")
started = time.time()
try:
connection2.execute("INSERT INTO test VALUES (2)")
except OperationalError as exception:
assert ustr(exception) == "database is locked"
assert time.time() - started >= 0.3
else:
self.fail("OperationalError not raised")
def _ensure_connected(self):
"""Ensure that we are connected to the database.
If the connection is marked as dead, or if we can't reconnect,
then raise DisconnectionError.
"""
if self._blocked:
raise ConnectionBlockedError("Access to connection is blocked")
if self._state == STATE_CONNECTED:
return
elif self._state == STATE_DISCONNECTED:
raise DisconnectionError("Already disconnected")
elif self._state == STATE_RECONNECT:
try:
self._raw_connection = self._database.raw_connect()
except DatabaseError as exc:
self._state = STATE_DISCONNECTED
self._raw_connection = None
raise DisconnectionError(ustr(exc))
else:
self._state = STATE_CONNECTED
def test_date_get_set_from_database():
date_str = "1977-05-04"
date_uni = ustr(date_str)
date_obj = date(1977, 5, 4)
datetime_obj = datetime(1977, 5, 4, 0, 0, 0)
variable = DateVariable()
variable.set(date_str, from_db=True)
assert variable.get() == date_obj
variable.set(date_uni, from_db=True)
assert variable.get() == date_obj
variable.set(date_obj, from_db=True)
assert variable.get() == date_obj
variable.set(datetime_obj, from_db=True)
assert variable.get() == date_obj
with pytest.raises(TypeError):
variable.set(0, from_db=True)
with pytest.raises(TypeError):
variable.set(marker, from_db=True)
with pytest.raises(ValueError):
variable.set("foobar", from_db=True)
def raw_execute(self, statement, params=None, _end=False):
"""Execute a raw statement with the given parameters.
This method will automatically retry on locked database errors.
This should be done by pysqlite, but it doesn't work with
versions < 2.3.4, so we make sure the timeout is respected
here.
"""
if _end:
self._in_transaction = False
elif not self._in_transaction:
# See story at the end to understand why we do BEGIN manually.
self._in_transaction = True
self._raw_connection.execute("BEGIN")
# Remember the time at which we started the operation. If pysqlite
# handles the timeout correctly, we won't retry the operation, because
# the timeout will have expired when the raw_execute() returns.
started = now()
while True:
try:
return Connection.raw_execute(self, statement, params)
except sqlite.OperationalError as e:
if ustr(e) != "database is locked":
raise
elif now() - started < self._database._timeout:
# pysqlite didn't handle the timeout correctly,
# so we sleep a little and then retry.
sleep(0.1)
else:
# The operation failed due to being unable to get a
# lock on the database. In this case, we are still
# in a transaction.
if _end:
self._in_transaction = True
raise
def is_disconnection_error(self, exc, extra_disconnection_errors=()):
# Attempt to use pgcode to determine the nature of the error. This is
# more reliable than string matching because it is not affected by
# locale settings. Fall through if pgcode is not available.
if isinstance(exc, Error):
pgcode = getattr(exc, "pgcode", None)
if pgcode in pg_connection_failure_codes:
return True
disconnection_errors = (
DatabaseError, InterfaceError, OperationalError,
ProgrammingError, extra_disconnection_errors)
if isinstance(exc, disconnection_errors):
# When the connection is closed by a termination of pgbouncer, a
# DatabaseError or subclass with no message (depending on
# psycopg2 version) is raised. If the raw connection is closed
# we assume it's actually a disconnection.
if isinstance(exc, DatabaseError):
if self._raw_connection.closed:
return True
msg = ustr(exc)
return (
"SSL SYSCALL error" in msg or
"EOF detected" in msg or
"connection already closed" in msg or
"connection not open" in msg or
"could not connect to server" in msg or
"could not receive data from server" in msg or
"could not send data to server" in msg or
"losed the connection unexpectedly" in msg or
"no connection to the server" in msg or
"server closed the connection unexpectedly" in msg or
"terminating connection due to administrator" in msg)
return False
def parse_get(self, value, to_db):
if to_db:
return ustr(value)
return value
def test_str_and_back(uri):
assert ustr(URI(uri)) == uri
def get_obj(self):
return ustr(self.id)