def connected():
"""Context manager that ensures we're connected to the database.
If there is not yet a connection to the database, this will connect on
entry and disconnect on exit. Preexisting connections will be left alone.
If the preexisting connection is not usable it is closed and a new
connection is made.
"""
if connection.connection is None:
connection.close_if_unusable_or_obsolete()
connection.ensure_connection()
try:
yield
finally:
connection.close()
elif connection.is_usable():
yield
else:
# Connection is not usable, so we disconnect and reconnect. Since
# the connection was previously connected we do not disconnect this
# new connection.
connection.close_if_unusable_or_obsolete()
connection.ensure_connection()
yield
def test_health_checks_enabled(self):
self.patch_settings_dict(conn_health_checks=True)
self.assertIsNone(connection.connection)
# Newly created connections are considered healthy without performing
# the health check.
with patch.object(connection, "is_usable", side_effect=AssertionError):
self.run_query()
old_connection = connection.connection
# Simulate request_finished.
connection.close_if_unusable_or_obsolete()
self.assertIs(old_connection, connection.connection)
# Simulate connection health check failing.
with patch.object(connection, "is_usable",
return_value=False) as mocked_is_usable:
self.run_query()
new_connection = connection.connection
# A new connection is established.
self.assertIsNot(new_connection, old_connection)
# Only one health check per "request" is performed, so the next
# query will carry on even if the health check fails. Next query
# succeeds because the real connection is healthy and only the
# health check failure is mocked.
self.run_query()
self.assertIs(new_connection, connection.connection)
self.assertEqual(mocked_is_usable.call_count, 1)
# Simulate request_finished.
connection.close_if_unusable_or_obsolete()
# The underlying connection is being reused further with health checks
# succeeding.
self.run_query()
self.run_query()
self.assertIs(new_connection, connection.connection)
def perform_job(self, *args, **kwargs):
"""
Handles connection (wait) timeouts on RQ.
If idle time of the worker exceeds wait timeout for connection to the
database (for MySQL it's 8 hours by default), connection is closed by
database server, but Django still thinks it has open connection to the
DB. This ends with errors like 'Lost connection to MySQL server' or
'MySQL server has gone away' (or similar for other backends).
Solution below fixes this bug by closing connections before and after
each job on worker and forcing Django to open a new one.
This comes in pair with `CONN_MAX_AGE` settings (https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-CONN_MAX_AGE). # noqa
To properly handle closing connection when using persistent connections
to the database, it's value should be lower than wait timeout of the
database server.
Resources:
* https://github.com/translate/pootle/issues/4094
* http://dev.mysql.com/doc/refman/5.7/en/gone-away.html
* https://dev.mysql.com/doc/refman/5.7/en/error-lost-connection.html
* http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_wait_timeout # noqa
"""
connection.close_if_unusable_or_obsolete()
result = super().perform_job(*args, **kwargs)
connection.close_if_unusable_or_obsolete()
return result
def __call__(self, context, message, *args, **kwargs):
"""Some basic extra filtering for directed commands"""
self.context = context
if context.target.startswith('#') != self.is_channel:
print " ! Failing channel"
return
if self.is_direct != (
message.startswith(context.ident.nick + ':') \
or message.endswith(context.ident.nick)
):
print " ! Failing directness"
return
connection.close_if_unusable_or_obsolete()
try:
translation.activate(self.get_language())
return self.run_command(context, *args, **kwargs)
except OperationalError as error:
if 'gone away' in str(error):
return "The database is being naughty, reconnecting..."
else:
return "A database error, hmmm."
except Exception:
if context:
context.connection.privmsg(context.target,
"There was an error")
raise
def signal_probe_change():
try:
cur = connection.cursor()
cur.execute('NOTIFY {}'.format(postgresql_channel))
connection.commit()
except Exception as db_err:
logger.error("Could not signal probe change: %s", db_err)
connection.close_if_unusable_or_obsolete()
def start(self):
if self.debug:
log.msg('Compare started')
d = defer.Deferred()
connection.close_if_unusable_or_obsolete()
d.addCallback(self.doCompare)
reactor.callLater(0, d.callback, None)
d.addBoth(self.finished)
return d
def check_connection():
"""
Due to known and seemingly unresolved issue with celery,
if a postgres connection drops and becomes unusable, it causes
failure of tasks and all their signal handlers that use the DB:
https://github.com/celery/celery/issues/621
"""
try:
connection.cursor()
except InterfaceError:
connection.close_if_unusable_or_obsolete()
connection.connect()
def run(self):
while True:
# LISTEN query
try:
cur = connection.cursor()
cur.execute('LISTEN {}'.format(postgresql_channel))
connection.commit()
except Exception as db_err:
connection.close_if_unusable_or_obsolete()
self.error_state = True
sleep_time = 2 * (1 + random.random())
logger.error(
"Could not execute the LISTEN query: %s. Sleep %ss.",
db_err, sleep_time)
time.sleep(sleep_time)
continue
# are we recovering from an error ?
if self.error_state:
# need to clear the probe_view. We might have missed some updates
probe_view = self.probe_view()
if probe_view:
logger.info("DB error recovery. Clear probe view.")
probe_view.clear()
else:
break
self.error_state = False
logger.info("Waiting for notifications on channel '%s'",
postgresql_channel)
pg_con = connection.connection
while True:
if select.select([pg_con], [], [], 5) == ([], [], []):
pass
else:
try:
pg_con.poll()
except Exception as db_err:
logger.error("Could not poll() the DB connection: %s",
db_err)
connection.close_if_unusable_or_obsolete()
break
if pg_con.notifies:
# clear notifications
while pg_con.notifies:
pg_con.notifies.pop()
logger.info("Received notification on channel '%s'",
postgresql_channel)
probe_view = self.probe_view()
if probe_view:
probe_view.clear()
else:
return
def update_cache_job(instance):
"""RQ job"""
job = get_current_job()
job_wrapper = JobWrapper(job.id, job.connection)
keys, decrement = job_wrapper.get_job_params()
# close unusable and obsolete connections before and after the job
# Note: setting CONN_MAX_AGE parameter can have negative side-effects
# CONN_MAX_AGE value should be lower than DB wait_timeout
connection.close_if_unusable_or_obsolete()
instance._update_cache_job(keys, decrement)
connection.close_if_unusable_or_obsolete()
job_wrapper.clear_job_params()
def test_set_autocommit_health_checks_enabled(self):
self.patch_settings_dict(conn_health_checks=True)
self.assertIsNone(connection.connection)
# Newly created connections are considered healthy without performing
# the health check.
with patch.object(connection, "is_usable", side_effect=AssertionError):
# Simulate outermost atomic block: changing autocommit for
# a connection.
connection.set_autocommit(False)
self.run_query()
connection.commit()
connection.set_autocommit(True)
old_connection = connection.connection
# Simulate request_finished.
connection.close_if_unusable_or_obsolete()
# Persistent connections are enabled.
self.assertIs(old_connection, connection.connection)
# Simulate connection health check failing.
with patch.object(
connection, "is_usable", return_value=False
) as mocked_is_usable:
# Simulate outermost atomic block: changing autocommit for
# a connection.
connection.set_autocommit(False)
new_connection = connection.connection
self.assertIsNot(new_connection, old_connection)
# Only one health check per "request" is performed, so a query will
# carry on even if the health check fails. This query succeeds
# because the real connection is healthy and only the health check
# failure is mocked.
self.run_query()
connection.commit()
connection.set_autocommit(True)
# The connection is unchanged.
self.assertIs(new_connection, connection.connection)
self.assertEqual(mocked_is_usable.call_count, 1)
# Simulate request_finished.
connection.close_if_unusable_or_obsolete()
# The underlying connection is being reused further with health checks
# succeeding.
connection.set_autocommit(False)
self.run_query()
connection.commit()
connection.set_autocommit(True)
self.assertIs(new_connection, connection.connection)
def test_health_checks_enabled_errors_occurred(self):
self.patch_settings_dict(conn_health_checks=True)
self.assertIsNone(connection.connection)
# Newly created connections are considered healthy without performing
# the health check.
with patch.object(connection, "is_usable", side_effect=AssertionError):
self.run_query()
old_connection = connection.connection
# Simulate errors_occurred.
connection.errors_occurred = True
# Simulate request_started (the connection is healthy).
connection.close_if_unusable_or_obsolete()
# Persistent connections are enabled.
self.assertIs(old_connection, connection.connection)
# No additional health checks after the one in
# close_if_unusable_or_obsolete() are executed during this "request"
# when running queries.
with patch.object(connection, "is_usable", side_effect=AssertionError):
self.run_query()
def update_cache_job(instance):
"""RQ job"""
# The script prefix needs to be set here because the generated
# URLs need to be aware of that and they are cached. Ideally
# Django should take care of setting this up, but it doesn't yet
# (fixed in Django 1.10):
# https://code.djangoproject.com/ticket/16734
script_name = u"/" if settings.FORCE_SCRIPT_NAME is None else force_unicode(settings.FORCE_SCRIPT_NAME)
set_script_prefix(script_name)
job = get_current_job()
job_wrapper = JobWrapper(job.id, job.connection)
keys, decrement = job_wrapper.get_job_params()
# close unusable and obsolete connections before and after the job
# Note: setting CONN_MAX_AGE parameter can have negative side-effects
# CONN_MAX_AGE value should be lower than DB wait_timeout
connection.close_if_unusable_or_obsolete()
instance._update_cache_job(keys, decrement)
connection.close_if_unusable_or_obsolete()
job_wrapper.clear_job_params()
def update_cache_job(instance):
"""RQ job"""
# The script prefix needs to be set here because the generated
# URLs need to be aware of that and they are cached. Ideally
# Django should take care of setting this up, but it doesn't yet
# (fixed in Django 1.10):
# https://code.djangoproject.com/ticket/16734
script_name = (u'/' if settings.FORCE_SCRIPT_NAME is None else
force_unicode(settings.FORCE_SCRIPT_NAME))
set_script_prefix(script_name)
job = get_current_job()
job_wrapper = JobWrapper(job.id, job.connection)
keys, decrement = job_wrapper.get_job_params()
# close unusable and obsolete connections before and after the job
# Note: setting CONN_MAX_AGE parameter can have negative side-effects
# CONN_MAX_AGE value should be lower than DB wait_timeout
connection.close_if_unusable_or_obsolete()
instance._update_cache_job(keys, decrement)
connection.close_if_unusable_or_obsolete()
job_wrapper.clear_job_params()
def test_health_checks_disabled(self):
self.patch_settings_dict(conn_health_checks=False)
self.assertIsNone(connection.connection)
# Newly created connections are considered healthy without performing
# the health check.
with patch.object(connection, "is_usable", side_effect=AssertionError):
self.run_query()
old_connection = connection.connection
# Simulate request_finished.
connection.close_if_unusable_or_obsolete()
# Persistent connections are enabled (connection is not).
self.assertIs(old_connection, connection.connection)
# Health checks are not performed.
with patch.object(connection, "is_usable", side_effect=AssertionError):
self.run_query()
# Health check wasn't performed and the connection is unchanged.
self.assertIs(old_connection, connection.connection)
self.run_query()
# The connection is unchanged after the next query either during
# the current "request".
self.assertIs(old_connection, connection.connection)
def _reset_db_connection():
"""
Reset database connection if it's unusable or obselete to avoid "connection already closed".
"""
connection.close_if_unusable_or_obsolete()
def submitBuildsets(self):
connection.close_if_unusable_or_obsolete()
log.msg('submitting %d pending buildsets' % len(self.pendings))
for tpl, changes in self.pendings.iteritems():
tree, locale = tpl
_t = self.trees[tree]
props = properties.Properties()
# figure out the latest change
try:
when = timeHelper(max(filter(None, (c.when for c in changes))))
except (ValueError, ImportError):
when = None
revisions = sorted(_t.branches.keys())
for k, v in _t.branches.iteritems():
_r = "000000000000"
if k == 'l10n':
repo = '%s/%s' % (v, locale)
else:
repo = v
try:
repo = Repository.objects.get(name=repo)
except Repository.DoesNotExist:
log.msg('Repository %s does not exist, skipping' % repo)
revisions.remove(k)
continue
q = Push.objects.filter(repository=repo,
changesets__branch__name='default')
if when is not None:
q = q.filter(push_date__lte=when)
try:
# get the latest changeset on the 'default' branch
# not strictly .tip, for pushes with heads on
# multiple branches (bug 602182)
_p = q.order_by('-pk')[0]
if _p.push_date:
if not when:
when = _p.push_date
else:
when = max(when, _p.push_date)
_c = _p.changesets.order_by('-pk')
_r = str(_c.filter(branch__name='default')[0].revision)
except IndexError:
# no pushes, try to get a good Changeset.
# this is guaranteed to at least return the null changeset
_r = str(
repo.changesets
.filter(branch__name='default')
.order_by('-pk')
.values_list('revision', flat=True)[0])
relpath = repo.relative_path()
props.setProperty(k+"_branch", relpath,
"Scheduler")
if relpath != repo.name:
props.setProperty("local_" + repo.name, relpath,
"Scheduler")
props.setProperty(k+"_revision", _r, "Scheduler")
_f = Forest.objects.get(name=_t.branches['l10n'])
# use the relative path of the en repo we got above
inipath = '{}/{}'.format(
props['en_branch'],
_t.l10ninis[_t.branches['en']][0])
props.update({"tree": tree,
"l10nbase": _f.relative_path(),
"locale": locale,
"inipath": inipath,
"srctime": when,
"revisions": revisions,
},
"Scheduler")
bs = buildset.BuildSet(self.builderNames,
SourceStamp(changes=changes),
properties=props)
self.submitBuildSet(bs)
log.msg('one buildset successfully submitted')
self.dSubmitBuildsets = None
self.pendings.clear()
def useable_connection():
connection.close_if_unusable_or_obsolete()
yield
connection.close_if_unusable_or_obsolete()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if not connection.in_atomic_block:
connection.close_if_unusable_or_obsolete()
def useable_connection():
connection.close_if_unusable_or_obsolete()
yield
connection.close_if_unusable_or_obsolete()
