def connected(): """Context manager that ensures we're connected to the database. If there is not yet a connection to the database, this will connect on entry and disconnect on exit. Preexisting connections will be left alone. If the preexisting connection is not usable it is closed and a new connection is made. """ if connection.connection is None: connection.close_if_unusable_or_obsolete() connection.ensure_connection() try: yield finally: connection.close() elif connection.is_usable(): yield else: # Connection is not usable, so we disconnect and reconnect. Since # the connection was previously connected we do not disconnect this # new connection. connection.close_if_unusable_or_obsolete() connection.ensure_connection() yield
def test_health_checks_enabled(self): self.patch_settings_dict(conn_health_checks=True) self.assertIsNone(connection.connection) # Newly created connections are considered healthy without performing # the health check. with patch.object(connection, "is_usable", side_effect=AssertionError): self.run_query() old_connection = connection.connection # Simulate request_finished. connection.close_if_unusable_or_obsolete() self.assertIs(old_connection, connection.connection) # Simulate connection health check failing. with patch.object(connection, "is_usable", return_value=False) as mocked_is_usable: self.run_query() new_connection = connection.connection # A new connection is established. self.assertIsNot(new_connection, old_connection) # Only one health check per "request" is performed, so the next # query will carry on even if the health check fails. Next query # succeeds because the real connection is healthy and only the # health check failure is mocked. self.run_query() self.assertIs(new_connection, connection.connection) self.assertEqual(mocked_is_usable.call_count, 1) # Simulate request_finished. connection.close_if_unusable_or_obsolete() # The underlying connection is being reused further with health checks # succeeding. self.run_query() self.run_query() self.assertIs(new_connection, connection.connection)
def perform_job(self, *args, **kwargs): """ Handles connection (wait) timeouts on RQ. If idle time of the worker exceeds wait timeout for connection to the database (for MySQL it's 8 hours by default), connection is closed by database server, but Django still thinks it has open connection to the DB. This ends with errors like 'Lost connection to MySQL server' or 'MySQL server has gone away' (or similar for other backends). Solution below fixes this bug by closing connections before and after each job on worker and forcing Django to open a new one. This comes in pair with `CONN_MAX_AGE` settings (https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-CONN_MAX_AGE). # noqa To properly handle closing connection when using persistent connections to the database, it's value should be lower than wait timeout of the database server. Resources: * https://github.com/translate/pootle/issues/4094 * http://dev.mysql.com/doc/refman/5.7/en/gone-away.html * https://dev.mysql.com/doc/refman/5.7/en/error-lost-connection.html * http://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_wait_timeout # noqa """ connection.close_if_unusable_or_obsolete() result = super().perform_job(*args, **kwargs) connection.close_if_unusable_or_obsolete() return result
def __call__(self, context, message, *args, **kwargs): """Some basic extra filtering for directed commands""" self.context = context if context.target.startswith('#') != self.is_channel: print " ! Failing channel" return if self.is_direct != ( message.startswith(context.ident.nick + ':') \ or message.endswith(context.ident.nick) ): print " ! Failing directness" return connection.close_if_unusable_or_obsolete() try: translation.activate(self.get_language()) return self.run_command(context, *args, **kwargs) except OperationalError as error: if 'gone away' in str(error): return "The database is being naughty, reconnecting..." else: return "A database error, hmmm." except Exception: if context: context.connection.privmsg(context.target, "There was an error") raise
def signal_probe_change(): try: cur = connection.cursor() cur.execute('NOTIFY {}'.format(postgresql_channel)) connection.commit() except Exception as db_err: logger.error("Could not signal probe change: %s", db_err) connection.close_if_unusable_or_obsolete()
def start(self): if self.debug: log.msg('Compare started') d = defer.Deferred() connection.close_if_unusable_or_obsolete() d.addCallback(self.doCompare) reactor.callLater(0, d.callback, None) d.addBoth(self.finished) return d
def check_connection(): """ Due to known and seemingly unresolved issue with celery, if a postgres connection drops and becomes unusable, it causes failure of tasks and all their signal handlers that use the DB: https://github.com/celery/celery/issues/621 """ try: connection.cursor() except InterfaceError: connection.close_if_unusable_or_obsolete() connection.connect()
def run(self): while True: # LISTEN query try: cur = connection.cursor() cur.execute('LISTEN {}'.format(postgresql_channel)) connection.commit() except Exception as db_err: connection.close_if_unusable_or_obsolete() self.error_state = True sleep_time = 2 * (1 + random.random()) logger.error( "Could not execute the LISTEN query: %s. Sleep %ss.", db_err, sleep_time) time.sleep(sleep_time) continue # are we recovering from an error ? if self.error_state: # need to clear the probe_view. We might have missed some updates probe_view = self.probe_view() if probe_view: logger.info("DB error recovery. Clear probe view.") probe_view.clear() else: break self.error_state = False logger.info("Waiting for notifications on channel '%s'", postgresql_channel) pg_con = connection.connection while True: if select.select([pg_con], [], [], 5) == ([], [], []): pass else: try: pg_con.poll() except Exception as db_err: logger.error("Could not poll() the DB connection: %s", db_err) connection.close_if_unusable_or_obsolete() break if pg_con.notifies: # clear notifications while pg_con.notifies: pg_con.notifies.pop() logger.info("Received notification on channel '%s'", postgresql_channel) probe_view = self.probe_view() if probe_view: probe_view.clear() else: return
def update_cache_job(instance): """RQ job""" job = get_current_job() job_wrapper = JobWrapper(job.id, job.connection) keys, decrement = job_wrapper.get_job_params() # close unusable and obsolete connections before and after the job # Note: setting CONN_MAX_AGE parameter can have negative side-effects # CONN_MAX_AGE value should be lower than DB wait_timeout connection.close_if_unusable_or_obsolete() instance._update_cache_job(keys, decrement) connection.close_if_unusable_or_obsolete() job_wrapper.clear_job_params()
def test_set_autocommit_health_checks_enabled(self): self.patch_settings_dict(conn_health_checks=True) self.assertIsNone(connection.connection) # Newly created connections are considered healthy without performing # the health check. with patch.object(connection, "is_usable", side_effect=AssertionError): # Simulate outermost atomic block: changing autocommit for # a connection. connection.set_autocommit(False) self.run_query() connection.commit() connection.set_autocommit(True) old_connection = connection.connection # Simulate request_finished. connection.close_if_unusable_or_obsolete() # Persistent connections are enabled. self.assertIs(old_connection, connection.connection) # Simulate connection health check failing. with patch.object( connection, "is_usable", return_value=False ) as mocked_is_usable: # Simulate outermost atomic block: changing autocommit for # a connection. connection.set_autocommit(False) new_connection = connection.connection self.assertIsNot(new_connection, old_connection) # Only one health check per "request" is performed, so a query will # carry on even if the health check fails. This query succeeds # because the real connection is healthy and only the health check # failure is mocked. self.run_query() connection.commit() connection.set_autocommit(True) # The connection is unchanged. self.assertIs(new_connection, connection.connection) self.assertEqual(mocked_is_usable.call_count, 1) # Simulate request_finished. connection.close_if_unusable_or_obsolete() # The underlying connection is being reused further with health checks # succeeding. connection.set_autocommit(False) self.run_query() connection.commit() connection.set_autocommit(True) self.assertIs(new_connection, connection.connection)
def test_health_checks_enabled_errors_occurred(self): self.patch_settings_dict(conn_health_checks=True) self.assertIsNone(connection.connection) # Newly created connections are considered healthy without performing # the health check. with patch.object(connection, "is_usable", side_effect=AssertionError): self.run_query() old_connection = connection.connection # Simulate errors_occurred. connection.errors_occurred = True # Simulate request_started (the connection is healthy). connection.close_if_unusable_or_obsolete() # Persistent connections are enabled. self.assertIs(old_connection, connection.connection) # No additional health checks after the one in # close_if_unusable_or_obsolete() are executed during this "request" # when running queries. with patch.object(connection, "is_usable", side_effect=AssertionError): self.run_query()
def update_cache_job(instance): """RQ job""" # The script prefix needs to be set here because the generated # URLs need to be aware of that and they are cached. Ideally # Django should take care of setting this up, but it doesn't yet # (fixed in Django 1.10): # https://code.djangoproject.com/ticket/16734 script_name = u"/" if settings.FORCE_SCRIPT_NAME is None else force_unicode(settings.FORCE_SCRIPT_NAME) set_script_prefix(script_name) job = get_current_job() job_wrapper = JobWrapper(job.id, job.connection) keys, decrement = job_wrapper.get_job_params() # close unusable and obsolete connections before and after the job # Note: setting CONN_MAX_AGE parameter can have negative side-effects # CONN_MAX_AGE value should be lower than DB wait_timeout connection.close_if_unusable_or_obsolete() instance._update_cache_job(keys, decrement) connection.close_if_unusable_or_obsolete() job_wrapper.clear_job_params()
def update_cache_job(instance): """RQ job""" # The script prefix needs to be set here because the generated # URLs need to be aware of that and they are cached. Ideally # Django should take care of setting this up, but it doesn't yet # (fixed in Django 1.10): # https://code.djangoproject.com/ticket/16734 script_name = (u'/' if settings.FORCE_SCRIPT_NAME is None else force_unicode(settings.FORCE_SCRIPT_NAME)) set_script_prefix(script_name) job = get_current_job() job_wrapper = JobWrapper(job.id, job.connection) keys, decrement = job_wrapper.get_job_params() # close unusable and obsolete connections before and after the job # Note: setting CONN_MAX_AGE parameter can have negative side-effects # CONN_MAX_AGE value should be lower than DB wait_timeout connection.close_if_unusable_or_obsolete() instance._update_cache_job(keys, decrement) connection.close_if_unusable_or_obsolete() job_wrapper.clear_job_params()
def test_health_checks_disabled(self): self.patch_settings_dict(conn_health_checks=False) self.assertIsNone(connection.connection) # Newly created connections are considered healthy without performing # the health check. with patch.object(connection, "is_usable", side_effect=AssertionError): self.run_query() old_connection = connection.connection # Simulate request_finished. connection.close_if_unusable_or_obsolete() # Persistent connections are enabled (connection is not). self.assertIs(old_connection, connection.connection) # Health checks are not performed. with patch.object(connection, "is_usable", side_effect=AssertionError): self.run_query() # Health check wasn't performed and the connection is unchanged. self.assertIs(old_connection, connection.connection) self.run_query() # The connection is unchanged after the next query either during # the current "request". self.assertIs(old_connection, connection.connection)
def _reset_db_connection(): """ Reset database connection if it's unusable or obselete to avoid "connection already closed". """ connection.close_if_unusable_or_obsolete()
def submitBuildsets(self): connection.close_if_unusable_or_obsolete() log.msg('submitting %d pending buildsets' % len(self.pendings)) for tpl, changes in self.pendings.iteritems(): tree, locale = tpl _t = self.trees[tree] props = properties.Properties() # figure out the latest change try: when = timeHelper(max(filter(None, (c.when for c in changes)))) except (ValueError, ImportError): when = None revisions = sorted(_t.branches.keys()) for k, v in _t.branches.iteritems(): _r = "000000000000" if k == 'l10n': repo = '%s/%s' % (v, locale) else: repo = v try: repo = Repository.objects.get(name=repo) except Repository.DoesNotExist: log.msg('Repository %s does not exist, skipping' % repo) revisions.remove(k) continue q = Push.objects.filter(repository=repo, changesets__branch__name='default') if when is not None: q = q.filter(push_date__lte=when) try: # get the latest changeset on the 'default' branch # not strictly .tip, for pushes with heads on # multiple branches (bug 602182) _p = q.order_by('-pk')[0] if _p.push_date: if not when: when = _p.push_date else: when = max(when, _p.push_date) _c = _p.changesets.order_by('-pk') _r = str(_c.filter(branch__name='default')[0].revision) except IndexError: # no pushes, try to get a good Changeset. # this is guaranteed to at least return the null changeset _r = str( repo.changesets .filter(branch__name='default') .order_by('-pk') .values_list('revision', flat=True)[0]) relpath = repo.relative_path() props.setProperty(k+"_branch", relpath, "Scheduler") if relpath != repo.name: props.setProperty("local_" + repo.name, relpath, "Scheduler") props.setProperty(k+"_revision", _r, "Scheduler") _f = Forest.objects.get(name=_t.branches['l10n']) # use the relative path of the en repo we got above inipath = '{}/{}'.format( props['en_branch'], _t.l10ninis[_t.branches['en']][0]) props.update({"tree": tree, "l10nbase": _f.relative_path(), "locale": locale, "inipath": inipath, "srctime": when, "revisions": revisions, }, "Scheduler") bs = buildset.BuildSet(self.builderNames, SourceStamp(changes=changes), properties=props) self.submitBuildSet(bs) log.msg('one buildset successfully submitted') self.dSubmitBuildsets = None self.pendings.clear()
def useable_connection(): connection.close_if_unusable_or_obsolete() yield connection.close_if_unusable_or_obsolete()
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not connection.in_atomic_block: connection.close_if_unusable_or_obsolete()