def afterTest(self, test):
"""
Clean up any changes to the test database.
"""
# Restore transaction support on tests
from django.conf import settings
from django.db import connections, transaction
from django.test.utils import setup_test_environment, teardown_test_environment
use_transaction_isolation = self._should_use_transaction_isolation(
test, settings)
if self._should_rebuild_schema(test):
for connection in connections.all():
connection.creation.destroy_test_db(
self.old_db, verbosity=self.verbosity)
teardown_test_environment()
setup_test_environment()
for connection in connections.all():
connection.creation.create_test_db(verbosity=self.verbosity)
self.restore_transaction_support(transaction)
transaction.commit()
if transaction.is_managed():
transaction.leave_transaction_management()
# If connection is not closed Postgres can go wild with
# character encodings.
for connection in connections.all():
connection.close()
logger.debug("Running syncdb")
self._num_syncdb_calls += 1
self._loaded_test_fixtures = []
return
if use_transaction_isolation:
self.restore_transaction_support(transaction)
logger.debug("Rolling back")
transaction.rollback()
if transaction.is_managed():
transaction.leave_transaction_management()
# If connection is not closed Postgres can go wild with
# character encodings.
for connection in connections.all():
connection.close()
else:
# Have to clear the db even if we're using django because django
# doesn't properly flush the database after a test. It relies on
# flushing before a test, so we want to avoid the case where a django
# test doesn't flush and then a normal test runs, because it will
# expect the db to already be flushed
self._flush_db()
self._loaded_test_fixtures = []
self.call_plugins_method('afterRollback', settings)
def run_appcfg(argv):
# We don't really want to use that one though, it just executes this one
from google.appengine.tools import appcfg
# Reset the logging level to WARN as appcfg will spew tons of logs on INFO
logging.getLogger().setLevel(logging.WARN)
new_args = argv[:]
new_args[1] = 'update'
if appconfig.runtime != 'python':
new_args.insert(1, '-R')
new_args.append(PROJECT_DIR)
syncdb = True
if '--nosyncdb' in new_args:
syncdb = False
new_args.remove('--nosyncdb')
appcfg.main(new_args)
if syncdb:
print 'Running syncdb.'
# Wait a little bit for deployment to finish
for countdown in range(9, 0, -1):
sys.stdout.write('%s\r' % countdown)
time.sleep(1)
from django.db import connections
for connection in connections.all():
if hasattr(connection, 'setup_remote'):
connection.setup_remote()
call_command('syncdb', remote=True, interactive=True)
if getattr(settings, 'ENABLE_PROFILER', False):
print '--------------------------\n' \
'WARNING: PROFILER ENABLED!\n' \
'--------------------------'
def _should_create_database(connection):
"""Return whether we should recreate the given DB.
This is true if the DB doesn't exist or the REUSE_DB env var isn't truthy.
"""
# TODO: Notice when the Model classes change and return True. Worst case,
# we can generate sqlall and hash it, though it's a bit slow (2 secs) and
# hits the DB for no good reason. Until we find a faster way, I'm inclined
# to keep making people explicitly saying REUSE_DB if they want to reuse
# the DB.
if not _can_support_reuse_db(connection):
return True
# Notice whether the DB exists, and create it if it doesn't:
try:
# Slight modification to fix proposed at
# https://github.com/jbalogh/django-nose/pull/101
# for bug https://github.com/jbalogh/django-nose/issues/76
for conn in connections.all():
conn.close()
connection.cursor()
except Exception: # TODO: Be more discerning but still DB agnostic.
return True
return not _reusing_db()
def make_view_atomic(self, view):
non_atomic_requests = getattr(view, '_non_atomic_requests', set())
for db in connections.all():
if (db.settings_dict['ATOMIC_REQUESTS']
and db.alias not in non_atomic_requests):
view = transaction.atomic(using=db.alias)(view)
return view
def _queue(self):
try:
redis = Db('eventadmindetail', self.get_object().id)
except:
redis = Db('eventadmindetail', self.event_obj.id)
pubsub = redis.object().pubsub()
pubsub.subscribe(redis.key)
for conn in connections.all():
conn.close()
while True:
for m in pubsub.listen():
if m['type'] == 'message':
msg = u"compatibility: true\n"
msg += u"retry: 10000\n"
msg += u"data: {}\n\n".format(m['data'])
yield msg
stream = u"compatibility: true\n"
stream += u"retry: 10000\n"
stream += u"data: {}\n\n".format(
json.dumps({"event": "stream"}))
yield stream
time.sleep(0.5)
def test_connections_thread_local(self):
"""
Ensure that the connections are different for each thread.
Refs #17258.
"""
# Map connections by id because connections with identical aliases
# have the same hash.
connections_dict = {}
for conn in connections.all():
connections_dict[id(conn)] = conn
def runner():
from django.db import connections
for conn in connections.all():
# Allow thread sharing so the connection can be closed by the
# main thread.
conn.allow_thread_sharing = True
connections_dict[id(conn)] = conn
for x in range(2):
t = threading.Thread(target=runner)
t.start()
t.join()
self.assertEqual(len(connections_dict), 6)
# Finish by closing the connections opened by the other threads (the
# connection opened in the main thread will automatically be closed on
# teardown).
for conn in connections_dict.values():
if conn is not connection:
conn.close()
def update_connections_time_zone(**kwargs):
if kwargs['setting'] == 'TIME_ZONE':
# Reset process time zone
if hasattr(time, 'tzset'):
if kwargs['value']:
os.environ['TZ'] = kwargs['value']
else:
os.environ.pop('TZ', None)
time.tzset()
# Reset local time zone cache
timezone.get_default_timezone.cache_clear()
# Reset the database connections' time zone
if kwargs['setting'] in {'TIME_ZONE', 'USE_TZ'}:
for conn in connections.all():
try:
del conn.timezone
except AttributeError:
pass
try:
del conn.timezone_name
except AttributeError:
pass
tz_sql = conn.ops.set_time_zone_sql()
if tz_sql and conn.timezone_name:
with conn.cursor() as cursor:
cursor.execute(tz_sql, [conn.timezone_name])
def update_connections_time_zone(**kwargs):
if kwargs['setting'] == 'TIME_ZONE':
# Reset process time zone
if hasattr(time, 'tzset'):
if kwargs['value']:
os.environ['TZ'] = kwargs['value']
else:
os.environ.pop('TZ', None)
time.tzset()
# Reset local time zone cache
timezone._localtime = None
# Reset the database connections' time zone
if kwargs['setting'] == 'USE_TZ' and settings.TIME_ZONE != 'UTC':
USE_TZ, TIME_ZONE = kwargs['value'], settings.TIME_ZONE
elif kwargs['setting'] == 'TIME_ZONE' and not settings.USE_TZ:
USE_TZ, TIME_ZONE = settings.USE_TZ, kwargs['value']
else:
# no need to change the database connnections' time zones
return
tz = 'UTC' if USE_TZ else TIME_ZONE
for conn in connections.all():
conn.settings_dict['TIME_ZONE'] = tz
tz_sql = conn.ops.set_time_zone_sql()
if tz_sql:
conn.cursor().execute(tz_sql, [tz])
def runner():
from django.db import connections
for conn in connections.all():
# Allow thread sharing so the connection can be closed by the
# main thread.
conn.allow_thread_sharing = True
connections_dict[id(conn)] = conn
def setup(self):
if self.should_skip_test_setup():
return
if self.optimize_migrations:
self.optimizer = optimize_apps_for_test_labels(self.test_labels)
self.optimizer.__enter__()
from corehq.blobs.tests.util import TemporaryFilesystemBlobDB
self.blob_db = TemporaryFilesystemBlobDB()
# get/verify list of apps with databases to be deleted on teardown
databases = getattr(settings, "COUCHDB_DATABASES", [])
if isinstance(databases, (list, tuple)):
# Convert old style to new style
databases = {app_name: uri for app_name, uri in databases}
self.apps = [self.verify_test_db(*item) for item in databases.items()]
if self.skip_setup_for_reuse_db:
from django.db import connections
old_names = []
for connection in connections.all():
db = connection.settings_dict
assert db["NAME"].startswith(TEST_DATABASE_PREFIX), db["NAME"]
try:
connection.ensure_connection()
except OperationalError:
break # cannot connect; resume normal setup
old_names.append((connection, db["NAME"], True))
else:
self.old_names = old_names, []
return # skip remaining setup
sys.__stdout__.write("\n") # newline for creating database message
super(HqdbContext, self).setup()
def update_connections_time_zone(**kwargs):
if kwargs["setting"] == "TIME_ZONE":
# Reset process time zone
if hasattr(time, "tzset"):
if kwargs["value"]:
os.environ["TZ"] = kwargs["value"]
else:
os.environ.pop("TZ", None)
time.tzset()
# Reset local time zone cache
timezone.get_default_timezone.cache_clear()
# Reset the database connections' time zone
if kwargs["setting"] in {"TIME_ZONE", "USE_TZ"}:
for conn in connections.all():
try:
del conn.timezone
except AttributeError:
pass
try:
del conn.timezone_name
except AttributeError:
pass
tz_sql = conn.ops.set_time_zone_sql()
if tz_sql:
with conn.cursor() as cursor:
cursor.execute(tz_sql, [conn.timezone_name])
def _should_create_database(connection):
"""Return whether we should recreate the given DB.
This is true if the DB doesn't exist or the REUSE_DB env var isn't truthy.
"""
# TODO: Notice when the Model classes change and return True. Worst case,
# we can generate sqlall and hash it, though it's a bit slow (2 secs) and
# hits the DB for no good reason. Until we find a faster way, I'm inclined
# to keep making people explicitly saying REUSE_DB if they want to reuse
# the DB.
for conn in connections.all():
conn.close()
conn.connection = None
if not _can_support_reuse_db(connection):
return True
# Notice whether the DB exists, and create it if it doesn't:
try:
connection.cursor()
except StandardError: # TODO: Be more discerning but still DB agnostic.
return True
return not _reusing_db()
def handle(self, *args, **options):
apps = settings.INSTALLED_APPS
additional_settings = ''
additional_urls = ''
for con in connections.all():
db_name = con.settings_dict.get('NAME')
hammer_db = con.settings_dict.get('HAMMER')
if hammer_db:
print 'Setting up database "{0}"'.format(db_name)
cur = con.cursor()
data_types_dict = cur.db.introspection.data_types_reverse
tables = cur.db.introspection.get_table_list(cur)
table_dict = {x:{} for x in tables}
installed_models = cur.db.introspection.installed_models(cur)
for t in tables:
print 'Introspecting table "{0}"'.format(t)
table_dict[t]['converted_name'] = cur.db.introspection.table_name_converter(t)
table_dict[t]['django_name'] = cur.db.introspection.django_table_names(t)
table_dict[t]['table_spec'] = cur.db.introspection.get_table_description(cur, t)
table_dict[t]['constraints'] = cur.db.introspection.get_constraints(cur, t)
table_dict[t]['relations'] = cur.db.introspection.get_relations(cur, t)
table_dict[t]['pk'] = cur.db.introspection.get_primary_key_column(cur, t)
table_dict[t]['key_columns'] = cur.db.introspection.get_key_columns(cur, t)
hammer = Hammer(data_types=data_types_dict, models=table_dict, app_name=con.alias, app_db=con.settings_dict)
additional_settings += hammer.setup()
additional_urls += " url(r'^%s/', include('%s.urls'))," % (con.alias, con.alias)
if len(additional_settings) > 0:
print "\n\nAdd the following lines to (the end of) settings.py\n\n{0}\n".format(''.join(additional_settings))
if len(additional_urls) > 0:
print "\nAdd the following lines to urls.py\n\n{0}\n".format(''.join(additional_urls))
def update_connections_time_zone(**kwargs):
if kwargs["setting"] == "TIME_ZONE":
# Reset process time zone
if hasattr(time, "tzset"):
if kwargs["value"]:
os.environ["TZ"] = kwargs["value"]
else:
os.environ.pop("TZ", None)
time.tzset()
# Reset local time zone cache
timezone.get_default_timezone.cache_clear()
# Reset the database connections' time zone
if kwargs["setting"] == "USE_TZ" and settings.TIME_ZONE != "UTC":
USE_TZ, TIME_ZONE = kwargs["value"], settings.TIME_ZONE
elif kwargs["setting"] == "TIME_ZONE" and not settings.USE_TZ:
USE_TZ, TIME_ZONE = settings.USE_TZ, kwargs["value"]
else:
# no need to change the database connnections' time zones
return
tz = "UTC" if USE_TZ else TIME_ZONE
for conn in connections.all():
conn.settings_dict["TIME_ZONE"] = tz
tz_sql = conn.ops.set_time_zone_sql()
if tz_sql:
conn.cursor().execute(tz_sql, [tz])
def process_response(request, response):
if hasattr(request, 'user') and is_authenticated(request):
response['User-Session'] = request.user.username
if django_settings.DEBUG:
# When DEBUG=False, Django will not store information regarding
# the SQL queries performed so there is not point here.
logger = logging.getLogger('django.db.backends')
nb_queries = 0
duration = timedelta()
for connection in connections.all():
nb_queries += len(connection.queries)
for query in connection.queries:
convert = datetime.strptime(query['time'], "%S.%f")
duration += timedelta(
0, convert.second, convert.microsecond)
# days, seconds, microseconds
if hasattr(request, 'starts_at'):
request_duration = datetime_or_now() - request.starts_at
logger.debug(
"%s %s executed %d SQL queries in %s (request duration: %s)",
request.method, request.get_full_path(),
nb_queries, duration, request_duration,
extra={'request': request,
'nb_queries': nb_queries,
'queries_duration': str(duration),
'request_duration': request_duration})
return response
def run_from_argv(self, argv):
from django.db import connections
for connection in connections.all():
if hasattr(connection, 'setup_remote'):
connection.setup_remote()
argv = argv[:1] + argv[2:]
execute_from_command_line(argv)
def databases(*args, **kwargs):
"""
Retrieve the stats data of each database.
:return: Stats data of each database.
"""
stats = {}
for connection in connections.all():
try:
connection.connect()
except:
is_usable = False
else:
is_usable = connection.is_usable()
finally:
stats[connection.alias] = {
'vendor': connection.vendor,
'is_usable': is_usable,
'allow_thread_sharing': connection.allow_thread_sharing,
'autocommit': connection.autocommit,
'commit_on_exit': connection.commit_on_exit,
'in_atomic_block': connection.in_atomic_block,
'settings': {k: v for k, v in connection.settings_dict.items() if k not in ('USER', 'PASSWORD')}
}
return stats
def test_bulk_sms_send_multiproc(self):
# close database connections manually to make sure they're not passed
# to subprocesses
for conn in django_db_connections.all():
conn.close()
# send the messages in parallel using 10 processes. use
# multiprocessing rather than threads so we don't have to clean
# up database connections that the threads might leave open.
cache.close()
pool = multiprocessing.Pool(10)
try:
for i in range(30):
BulkMessageFactory(batch=self.batch, sms=None)
for j in range(10):
BulkMessageFactory(batch=self.batch, sms=None, deleted=True)
# 40 is the first multiple of 10 greater than or equal to 31
num_sent = send_messages(self.batch, num_to_send=40, map_=pool.map)
batch = Batch.objects.get(pk=self.batch.pk)
self.assertEqual(batch.errors, 0)
self.assertEqual(batch.status, Batch.COMPLETED)
# assert that we report the right number sent too
self.assertEqual(num_sent, 30)
self.assertEqual(len(self.outbound), 30)
finally:
pool.terminate()
pool.join()
def _post_teardown(self):
"""Performs any post-test things. This includes:
* Flushing the contents of the database, to leave a clean slate. If
the class has an 'available_apps' attribute, post_migrate isn't fired.
* Force-closing the connection, so the next test gets a clean cursor.
"""
try:
self._fixture_teardown()
super(TransactionTestCase, self)._post_teardown()
# Some DB cursors include SQL statements as part of cursor
# creation. If you have a test that does rollback, the effect of
# these statements is lost, which can effect the operation of
# tests (e.g., losing a timezone setting causing objects to be
# created with the wrong time). To make sure this doesn't happen,
# get a clean connection at the start of every test.
for conn in connections.all():
conn.close()
finally:
if self.available_apps is not None:
apps.unset_available_apps()
setting_changed.send(sender=settings._wrapped.__class__,
setting='INSTALLED_APPS',
value=settings.INSTALLED_APPS,
enter=False)
def handle(self, *args, **options):
for connection in connections.all():
with connection.cursor() as cursor:
for model in apps.get_models():
if issubclass(model, models.ShardedModel):
continue
self.run(connection, cursor, model, **options)
def connections_support_transactions():
"""
Returns True if all connections support transactions. This is messy
because 2.4 doesn't support any or all.
"""
return all(conn.features.supports_transactions
for conn in connections.all())
def connections_support_transactions():
"""
Returns True if all connections support transactions. This is messy
because 2.4 doesn't support any or all.
"""
return all(conn.settings_dict['SUPPORTS_TRANSACTIONS']
for conn in connections.all())
def run_appcfg(argv):
# We don't really want to use that one though, it just executes this one
from google.appengine.tools import appcfg
# Reset the logging level to WARN as appcfg will spew tons of logs on INFO
logging.getLogger().setLevel(logging.WARN)
new_args = argv[:]
new_args[1] = "update"
new_args.append(PROJECT_DIR)
syncdb = True
if "--nosyncdb" in new_args:
syncdb = False
new_args.remove("--nosyncdb")
appcfg.main(new_args)
if syncdb:
print "Running syncdb."
# Wait a little bit for deployment to finish
for countdown in range(9, 0, -1):
sys.stdout.write("%s\r" % countdown)
time.sleep(1)
from django.db import connections
for connection in connections.all():
if hasattr(connection, "setup_remote"):
connection.setup_remote()
call_command("syncdb", remote=True, interactive=True)
if getattr(settings, "ENABLE_PROFILER", False):
print "--------------------------\n" "WARNING: PROFILER ENABLED!\n" "--------------------------"
def __init__(self, addr):
try:
from django.test.testcases import LiveServerThread
except ImportError:
pytest.skip('live_server tests not supported in Django < 1.4')
from django.db import connections
connections_override = {}
for conn in connections.all():
# If using in-memory sqlite databases, pass the connections to
# the server thread.
if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3'
and conn.settings_dict['NAME'] == ':memory:'):
# Explicitly enable thread-shareability for this connection
conn.allow_thread_sharing = True
connections_override[conn.alias] = conn
liveserver_kwargs = {'connections_override': connections_override}
try:
from django.test.testcases import _StaticFilesHandler
liveserver_kwargs['static_handler'] = _StaticFilesHandler
except ImportError:
pass
host, possible_ports = parse_addr(addr)
self.thread = LiveServerThread(host, possible_ports,
**liveserver_kwargs)
self.thread.daemon = True
self.thread.start()
self.thread.is_ready.wait()
if self.thread.error:
raise self.thread.error
def target(pk):
# wait until all threads have started before attempting to send
trigger.wait()
send_message_by_id(pk)
# workaround https://code.djangoproject.com/ticket/22420
for conn in django_db_connections.all():
conn.close()
def monkey_patch_creation_for_db_reuse():
from django.db import connections
for connection in connections.all():
if test_database_exists_from_previous_run(connection):
_monkeypatch(connection.creation, 'create_test_db',
create_test_db_with_reuse)
def update_connections_time_zone(**kwargs):
if kwargs['setting'] == 'TIME_ZONE':
# Reset process time zone
if hasattr(time, 'tzset'):
if kwargs['value']:
os.environ['TZ'] = kwargs['value']
else:
os.environ.pop('TZ', None)
time.tzset()
# Reset local time zone cache
timezone.get_default_timezone.cache_clear()
# Reset the database connections' time zone
if kwargs['setting'] in {'TIME_ZONE', 'USE_TZ'}:
for conn in connections.all():
try:
del conn.timezone
except AttributeError:
pass
try:
del conn.timezone_name
except AttributeError:
pass
conn.ensure_timezone()
def test_connections_thread_local(self):
"""
Ensure that the connections are different for each thread.
Refs #17258.
"""
connections_set = set()
for conn in connections.all():
connections_set.add(conn)
def runner():
from django.db import connections
for conn in connections.all():
# Allow thread sharing so the connection can be closed by the
# main thread.
conn.allow_thread_sharing = True
connections_set.add(conn)
for x in range(2):
t = threading.Thread(target=runner)
t.start()
t.join()
self.assertEqual(len(connections_set), 6)
# Finish by closing the connections opened by the other threads (the
# connection opened in the main thread will automatically be closed on
# teardown).
for conn in connections_set:
if conn is not connection:
conn.close()
def _should_create_database(connection):
"""Return whether we should recreate the given DB.
This is true if the DB doesn't exist or the REUSE_DB env var isn't truthy.
"""
# TODO: Notice when the Model classes change and return True. Worst case,
# we can generate sqlall and hash it, though it's a bit slow (2 secs) and
# hits the DB for no good reason. Until we find a faster way, I'm inclined
# to keep making people explicitly saying REUSE_DB if they want to reuse
# the DB.
if not _can_support_reuse_db(connection):
return True
# Notice whether the DB exists, and create it if it doesn't:
try:
# Connections are cached by some backends, if other code has connected
# to the database previously under a different database name the
# cached connection will be used and no exception will be raised.
# Avoiding this by closing connections and setting to null
for connection in connections.all():
connection.close()
connection.connection = None
connection.cursor()
except Exception: # TODO: Be more discerning but still DB agnostic.
return True
return not _reusing_db()
def setUpClass(cls):
connections_override = {}
for conn in connections.all():
# If using in-memory sqlite databases, pass the connections to
# the server thread.
if (conn.vendor == 'sqlite' and
conn.settings_dict['NAME'] == ':memory:'):
# Explicitly enable thread-shareability for this connection
conn.allow_thread_sharing = True
connections_override[conn.alias] = conn
# Launch the live server's thread
specified_address = os.environ.get(
'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081')
# The specified ports may be of the form '8000-8010,8080,9200-9300'
# i.e. a comma-separated list of ports or ranges of ports, so we break
# it down into a detailed list of all possible ports.
possible_ports = []
try:
host, port_ranges = specified_address.split(':')
for port_range in port_ranges.split(','):
# A port range can be of either form: '8000' or '8000-8010'.
extremes = list(map(int, port_range.split('-')))
assert len(extremes) in [1, 2]
if len(extremes) == 1:
# Port range of the form '8000'
possible_ports.append(extremes[0])
else:
# Port range of the form '8000-8010'
for port in range(extremes[0], extremes[1] + 1):
possible_ports.append(port)
except Exception:
msg = 'Invalid address ("%s") for live server.' % specified_address
six.reraise(
ImproperlyConfigured,
ImproperlyConfigured(msg),
sys.exc_info()[2]
)
cls.server_thread = LiveTornadoThread(
host,
possible_ports,
cls.static_handler,
connections_override=connections_override
)
cls.server_thread.daemon = True
cls.server_thread.start()
# Wait for the live server to be ready
cls.server_thread.is_ready.wait()
if cls.server_thread.error:
# Clean up behind ourselves, since tearDownClass won't get called
# in case of errors.
cls._tearDownClassInternal()
raise cls.server_thread.error
cls.live_server_url = 'http://%s:%s' % (
cls.server_thread.host, cls.server_thread.port)
super(LiveTornadoTestCase, cls).setUpClass()
def monkey_patch_creation_for_db_suffix(suffix=None):
from django.db import connections
if suffix is not None:
def _get_test_db_name(self):
"""
Internal implementation - returns the name of the test DB that will be
created. Only useful when called from create_test_db() and
_create_test_db() and when no external munging is done with the 'NAME'
or 'TEST_NAME' settings.
"""
db_name = _get_db_name(self.connection.settings_dict, suffix)
return db_name
for connection in connections.all():
_monkeypatch(connection.creation, '_get_test_db_name', _get_test_db_name)
def post_report(test_object):
try:
c = Client()
login_ret = c.login(email='*****@*****.**',
password='******')
test_object.assertTrue(login_ret, 'Client is not logged in.')
response = c.post(reverse(
'realtime:earthquake_report_list',
kwargs={'shake_id': u'20150619200628'}),
report_multipart,
format='multipart')
test_object.assertEqual(response.status_code,
status.HTTP_200_OK)
except Exception as e:
for conn in connections.all():
conn.close()
raise e
def _pre_setup(self):
for connection in connections.all():
if self._is_in_memory_db(connection):
raise ImproperlyConfigured(
"ChannelLiveServerTestCase can not be used with in memory databases"
)
super(ChannelsLiveServerTestCase, self)._pre_setup()
self._live_server_modified_settings = modify_settings(
ALLOWED_HOSTS={"append": self.host})
self._live_server_modified_settings.enable()
self._server_process = self.ProtocolServerProcess(
self.host, get_default_application())
self._server_process.start()
self._server_process.ready.wait()
self._port = self._server_process.port.value
def handle(self, *args, **options):
self.stdout.write("")
self.stdout.write("_" * 79)
self.stdout.write(self.help)
self.stdout.write("")
# self.stdout.write("settings.DATABASES = ", ending="")
# pprint(settings.DATABASES)
# self.stdout.write("-"*79)
for alias, settings_dict in settings.DATABASES.items():
self.stdout.write("Database alias %r:" % alias)
self.stdout.write("")
engine = settings_dict["ENGINE"]
engine = engine.rsplit(".", 1)[-1]
self.stdout.write("engine...............: %r" % engine)
if engine == "sqlite3":
# https://docs.python.org/3/library/sqlite3.html#module-functions-and-constants
import sqlite3
self.stdout.write("sqlite lib version...: %r" %
sqlite3.sqlite_version)
self.stdout.write("sqlite module version: %r" %
sqlite3.version)
self.stdout.write("name.................: %r" %
settings_dict["NAME"])
self.stdout.write("user.................: %r" %
settings_dict["USER"])
self.stdout.write("host.................: %r" %
settings_dict["HOST"])
self.stdout.write("port.................: %r" %
settings_dict["PORT"])
connection_list = connections.all()
self.stdout.write("")
self.stdout.write("There are %i connections." % len(connection_list))
for no, conn in enumerate(connection_list, 1):
self.stdout.write("")
self.stdout.write("connection %i alias: %r settings_dict:" %
(no, conn.alias))
pprint(conn.settings_dict)
self.stdout.write("")
self.stdout.write("-" * 79)
def ensure_sql_instrumented():
global sql_instrumented
if sql_instrumented:
return
sql_instrumented = True
if django.VERSION >= (2, 0):
for connection in connections.all():
install_db_execute_hook(connection=connection)
connection_created.connect(install_db_execute_hook)
logger.debug("Installed DB connection created signal handler")
else:
CursorWrapper.execute = execute_wrapper(CursorWrapper.execute)
CursorWrapper.executemany = executemany_wrapper(CursorWrapper.executemany)
logger.debug("Monkey patched SQL")
def prepare(self):
if connection.connection and not connection.is_usable():
# destroy the default mysql connection
# after this line, when you use ORM methods
# django will reconnect to the default mysql
del connections._connections.default
for c in connections.all():
try:
c._commit()
except:
pass
return
def process_request(self, request):
"""Initialize statistics variables and environment.
:return: Response object or None
:rtype: :class:`django.http.HttpResponse` or None
"""
self.is_active = self.can_process_request(request)
if self.is_active:
# Force DB connection to debug mode to get SQL time and number of SQL queries
for conn in connections.all():
conn.force_debug_cursor = True
self.initial_sql_count += len(conn.queries)
self.initial_sql_time += sum(
float(q["time"]) for q in conn.queries)
self.start_time = default_timer()
def __init__(self, addr: str) -> None:
from django.db import connections
from django.test.testcases import LiveServerThread
from django.test.utils import modify_settings
liveserver_kwargs = {} # type: Dict[str, Any]
connections_override = {}
for conn in connections.all():
# If using in-memory sqlite databases, pass the connections to
# the server thread.
if (conn.settings_dict["ENGINE"] == "django.db.backends.sqlite3"
and conn.settings_dict["NAME"] == ":memory:"):
# Explicitly enable thread-shareability for this connection
conn.allow_thread_sharing = True
connections_override[conn.alias] = conn
liveserver_kwargs["connections_override"] = connections_override
from django.conf import settings
if "django.contrib.staticfiles" in settings.INSTALLED_APPS:
from django.contrib.staticfiles.handlers import StaticFilesHandler
liveserver_kwargs["static_handler"] = StaticFilesHandler
else:
from django.test.testcases import _StaticFilesHandler
liveserver_kwargs["static_handler"] = _StaticFilesHandler
try:
host, port = addr.split(":")
except ValueError:
host = addr
else:
liveserver_kwargs["port"] = int(port)
self.thread = LiveServerThread(host, **liveserver_kwargs)
self._live_server_modified_settings = modify_settings(
ALLOWED_HOSTS={"append": host})
self.thread.daemon = True
self.thread.start()
self.thread.is_ready.wait()
if self.thread.error:
raise self.thread.error
def handle(self, *args, **options):
self.stdout.write("")
self.stdout.write("_" * 79)
self.stdout.write(self.help)
self.stdout.write("")
# self.stdout.write("settings.DATABASES = ", ending="")
# pprint(settings.DATABASES)
# self.stdout.write("-"*79)
for alias, settings_dict in settings.DATABASES.items():
self.stdout.write(f"Database alias {alias!r}:")
self.stdout.write("")
engine = settings_dict["ENGINE"]
engine = engine.rsplit(".", 1)[-1]
self.stdout.write(f"engine...............: {engine!r}")
if engine == "sqlite3":
# https://docs.python.org/3/library/sqlite3.html#module-functions-and-constants
import sqlite3
self.stdout.write(
f"sqlite lib version...: {sqlite3.sqlite_version!r}")
self.stdout.write(
f"sqlite module version: {sqlite3.version!r}")
self.stdout.write(
f"name.................: {settings_dict['NAME']!r}")
self.stdout.write(
f"user.................: {settings_dict['USER']!r}")
self.stdout.write(
f"host.................: {settings_dict['HOST']!r}")
self.stdout.write(
f"port.................: {settings_dict['PORT']!r}")
connection_list = connections.all()
self.stdout.write("")
self.stdout.write("There are %i connections." % len(connection_list))
for no, conn in enumerate(connection_list, 1):
self.stdout.write("")
self.stdout.write(
f"connection {no:d} alias: {conn.alias!r} settings_dict:")
pprint(conn.settings_dict)
self.stdout.write("")
self.stdout.write("-" * 79)
def setUpClass(cls):
connections_override = {}
for conn in connections.all():
# If using in-memory sqlite databases, pass the connections to
# the server thread.
if (conn.settings_dict['ENGINE'].rsplit('.', 1)[-1] in ('sqlite3', 'spatialite')
and conn.settings_dict['NAME'] == ':memory:'):
# Explicitly enable thread-shareability for this connection
conn.allow_thread_sharing = True
connections_override[conn.alias] = conn
# Launch the live server's thread
specified_address = os.environ.get(
'DJANGO_LIVE_TEST_SERVER_ADDRESS', 'localhost:8081')
# The specified ports may be of the form '8000-8010,8080,9200-9300'
# i.e. a comma-separated list of ports or ranges of ports, so we break
# it down into a detailed list of all possible ports.
possible_ports = []
try:
host, port_ranges = specified_address.split(':')
for port_range in port_ranges.split(','):
# A port range can be of either form: '8000' or '8000-8010'.
extremes = list(map(int, port_range.split('-')))
assert len(extremes) in [1, 2]
if len(extremes) == 1:
# Port range of the form '8000'
possible_ports.append(extremes[0])
else:
# Port range of the form '8000-8010'
for port in range(extremes[0], extremes[1] + 1):
possible_ports.append(port)
except Exception:
raise ImproperlyConfigured('Invalid address ("%s") for live '
'server.' % specified_address)
cls.server_thread = LiveServerThread(
host, possible_ports, connections_override)
cls.server_thread.daemon = True
cls.server_thread.start()
# Wait for the live server to be ready
cls.server_thread.is_ready.wait()
if cls.server_thread.error:
raise cls.server_thread.error
super(LiveServerTestCase, cls).setUpClass()
def prepare_task_execution():
"""
Clearing of old database connections for CONN_MAX_AGE option (database connection settings)
"""
if not TaskUtils.is_celery_worker():
return
try:
if TaskUtils.__connection_initialization_finished:
close_old_connections()
else:
for conn in connections.all():
conn.close()
TaskUtils.__connection_initialization_finished = True
except Exception:
pass
def _post_teardown(self):
""" Performs any post-test things. This includes:
* Putting back the original ROOT_URLCONF if it was changed.
* Force closing the connection, so that the next test gets
a clean cursor.
"""
self._fixture_teardown()
self._urlconf_teardown()
# Some DB cursors include SQL statements as part of cursor
# creation. If you have a test that does rollback, the effect
# of these statements is lost, which can effect the operation
# of tests (e.g., losing a timezone setting causing objects to
# be created with the wrong time).
# To make sure this doesn't happen, get a clean connection at the
# start of every test.
for conn in connections.all():
conn.close()
def _count_queries(self, which):
for c in connections.all():
for q in c.queries:
if not self._ignore_sql(q):
if q.get('sql') and self.READ_QUERY_REGEX.search(q['sql']) is not None:
self.stats[which][c.alias]['reads'] += 1
else:
self.stats[which][c.alias]['writes'] += 1
self.stats[which][c.alias]['total'] += 1
self.queries[q['sql']] += 1
# We'll show the worst offender; i.e. the query with the most duplicates
duplicates = self.queries.most_common(1)
if duplicates:
sql, count = duplicates[0]
self.stats[which][c.alias]['duplicates'] = count
else:
self.stats[which][c.alias]['duplicates'] = 0
def monkey_patch_create_with_template():
"""
Monkeypatch create sql to do 'CREATE DATABASE test_xxx WITH TEMPLATE xxx' instead of just 'CREATE DATABASE test_xxx'
Useful when testing with PostgreSQL and barebone of existing database
"""
def _sql_table_creation_suffix(self):
try:
return " WITH TEMPLATE=%s" % self.connection.settings_dict[
'TEST_WITH_TEMPLATE']
except KeyError:
return ""
from django.db import connections
for connection in connections.all():
if "TEST_WITH_TEMPLATE" in connection.settings_dict:
_monkeypatch(connection.creation, 'sql_table_creation_suffix',
_sql_table_creation_suffix)
def _fixture_teardown(self):
if not connections_support_transactions():
return super(TestCase, self)._fixture_teardown()
# If the test case has a multi_db=True flag, teardown all databases.
# Otherwise, just teardown default.
if getattr(self, 'multi_db', False):
databases = connections
else:
databases = [DEFAULT_DB_ALIAS]
restore_transaction_methods()
for db in databases:
transaction.rollback(using=db)
transaction.leave_transaction_management(using=db)
for connection in connections.all():
connection.close()
def _report_queries(self, descr=None):
if not self.enable_report_queries:
return
if not hasattr(self, 'start_time'):
return
end_time = monotonic.monotonic()
if descr is None:
descr = ""
nb_queries = 0
duration = timedelta()
for conn in connections.all():
nb_queries += len(conn.queries)
for query in conn.queries:
convert = datetime.strptime(query['time'], "%S.%f")
duration += timedelta(0, convert.second, convert.microsecond)
# days, seconds, microseconds
LOGGER.debug("(elapsed: %.2fs) %s: %s for %d SQL queries",
(end_time - self.start_time), descr, duration, nb_queries)
def django_db_setup(django_db_blocker):
from django.conf import settings
settings.DATABASES['default']['NAME'] = 'clima_test'
run_sql('DROP DATABASE IF EXISTS clima_test')
run_sql('CREATE DATABASE clima_test TEMPLATE clima_template')
with django_db_blocker.unblock():
call_command('loaddata',
'app/tests/functional/fixtures/climato_data.json')
yield
for connection in connections.all():
connection.close()
run_sql('DROP DATABASE clima_test')
def _pre_setup(self):
for connection in connections.all():
if self._is_in_memory_db(connection):
raise ImproperlyConfigured(
'ChannelLiveServerTestCase can not be used with in memory databases'
)
channel_layers = ChannelLayerManager()
if len(channel_layers.configs) > 1:
raise ImproperlyConfigured(
'ChannelLiveServerTestCase does not support multiple CHANNEL_LAYERS at this time'
)
channel_layer = channel_layers.make_test_backend(DEFAULT_CHANNEL_LAYER)
if 'flush' in channel_layer.extensions:
channel_layer.flush()
super(ChannelLiveServerTestCase, self)._pre_setup()
server_ready = multiprocessing.Event()
self._port_storage = multiprocessing.Value('i')
self._server_process = self.ProtocolServerProcess(
self.host,
self._port_storage,
server_ready,
self._overridden_settings,
self._modified_settings,
connections.databases,
)
self._server_process.start()
server_ready.wait()
worker_ready = multiprocessing.Event()
self._worker_process = self.WorkerProcess(
worker_ready,
self.worker_threads,
self._overridden_settings,
self._modified_settings,
connections.databases,
self.serve_static,
)
self._worker_process.start()
worker_ready.wait()
def __init__(self, addr):
import django
from django.db import connections
from django.test.testcases import LiveServerThread
from django.test.utils import modify_settings
connections_override = {}
for conn in connections.all():
# If using in-memory sqlite databases, pass the connections to
# the server thread.
if (conn.settings_dict['ENGINE'] == 'django.db.backends.sqlite3' and
conn.settings_dict['NAME'] == ':memory:'):
# Explicitly enable thread-shareability for this connection
conn.allow_thread_sharing = True
connections_override[conn.alias] = conn
liveserver_kwargs = {'connections_override': connections_override}
from django.conf import settings
if 'django.contrib.staticfiles' in settings.INSTALLED_APPS:
from django.contrib.staticfiles.handlers import StaticFilesHandler
liveserver_kwargs['static_handler'] = StaticFilesHandler
else:
from django.test.testcases import _StaticFilesHandler
liveserver_kwargs['static_handler'] = _StaticFilesHandler
if django.VERSION < (1, 11):
host, possible_ports = parse_addr(addr)
self.thread = LiveServerThread(host, possible_ports,
**liveserver_kwargs)
else:
host = addr
self.thread = LiveServerThread(host, **liveserver_kwargs)
self._live_server_modified_settings = modify_settings(
ALLOWED_HOSTS={'append': host},
)
self._live_server_modified_settings.enable()
self.thread.daemon = True
self.thread.start()
self.thread.is_ready.wait()
if self.thread.error:
raise self.thread.error
def test_database_patch(self):
# We want to test that a connection-recreation event causes connections
# to get repatched. However since django tests are a atomic transaction
# we can't change the connection. Instead we test that the connection
# does get repatched if it's not patched.
for conn in connections.all():
unpatch_conn(conn)
# ensures that the internals are properly traced
url = reverse('users-list')
response = self.client.get(url)
eq_(response.status_code, 200)
# We would be missing span #3, the database span, if the connection
# wasn't patched.
spans = self.tracer.writer.pop()
eq_(len(spans), 3)
eq_(spans[0].name, 'django.request')
eq_(spans[1].name, 'django.template')
eq_(spans[2].name, 'sqlite.query')
def sirivm(self, message):
try:
with beeline.tracer(name="sirivm"):
if self.command is None:
self.command = import_bod_avl.Command().do_source()
response_timestamp = parse_datetime(message["when"])
beeline.add_context({
"items_count": len(message["items"]),
"age": (now() - response_timestamp).total_seconds()
})
vehicle_cache_keys = [self.command.get_vehicle_cache_key(item) for item in message["items"]]
with beeline.tracer(name="cache get many"):
vehicle_ids = cache.get_many(vehicle_cache_keys) # code: id
with beeline.tracer(name="vehicles in bulk"):
vehicles = self.command.vehicles.in_bulk(vehicle_ids.values()) # id: vehicle
self.command.vehicle_cache = { # code: vehicle
key: vehicles[vehicle_id] for key, vehicle_id in vehicle_ids.items() if vehicle_id in vehicles
}
with beeline.tracer(name="handle items"):
for item in message["items"]:
self.command.handle_item(item, response_timestamp)
with beeline.tracer(name="save"):
db_wrapper = HoneyDBWrapper()
with ExitStack() as stack:
for connection in connections.all():
stack.enter_context(connection.execute_wrapper(db_wrapper))
self.command.save()
with beeline.tracer(name="set many"):
cache.set_many({
key: value
for key, value in self.command.vehicle_id_cache.items()
if key not in vehicle_ids or value != vehicle_ids[key]
}, 43200)
except Exception as e:
capture_exception(e)
raise Exception
def atomic_update_or_create_with_close(
lock,
job_id: str,
run_time: datetime,
status: str,
exception: str = None,
traceback: str = None,
) -> "DjangoJobExecution":
job_execution = _old_atomic_update_or_create(
lock,
job_id,
run_time,
status,
exception,
traceback,
)
for conn in connections.all():
conn.close_if_unusable_or_obsolete()
return job_execution
def wrapper(*args, **kwargs):
# To record.
_to_record = False
if tracking and 'corelib.asynctask.async_api' in settings.INSTALLED_APPS:
_to_record = True
uuid = kwargs.pop('__async_task_uuid__')
func_name = kwargs.pop('__async_func_name__')
from corelib.asynctask.async_api.models import AsyncTask
obj = AsyncTask.objects.filter(uuid=uuid).first()
obj.status = 1
obj.save()
try:
# To do the real work.
res = func(*args, **kwargs)
except Exception as e:
if _to_record:
obj.result = False
obj.result_data = {
'result': False,
'error_msg':
f"ERROR: To run asynctask '{func_name}' failed. task uuid: '{uuid}'. {str(e)}",
'data': None
}
else:
raise e
else:
if _to_record:
obj.result = True
obj.return_data = {'result': True, 'data': res}
finally:
if _to_record:
obj.status = 2
obj.finish_time = timezone.now()
obj.save()
# To close db connections the way like handling django request_finished.
for conn in connections.all():
conn.close_if_unusable_or_obsolete()
return res
def django_db_modify_db_settings(django_db_keepdb):
from django.conf import settings
testdb_kwargs = {}
if django_db_keepdb:
testdb_kwargs["base_dir"] = os.path.join(
os.getcwd(), TESTING_POSTGRESQL_KEEP_DB_PATH)
with testing.postgresql.Postgresql(**testdb_kwargs) as postgresql:
db_url = postgresql.url()
settings.DATABASES["default"] = dj_database_url.parse(db_url)
settings.DATABASES["default"]["TEST"] = {
"NAME": settings.DATABASES["default"]["NAME"]
}
yield
for connection in connections.all():
connection.close()
def run():
ppid = os.getppid()
logger.warn(f'periodic beat started')
while True:
if os.getppid() != ppid:
# if the parent PID changes, this process has been orphaned
# via e.g., segfault or sigkill, we should exit too
pid = os.getpid()
logger.warn(f'periodic beat exiting gracefully pid:{pid}')
raise SystemExit()
try:
for conn in connections.all():
# If the database connection has a hiccup, re-establish a new
# connection
conn.close_if_unusable_or_obsolete()
self.run_pending()
except Exception:
logger.exception(
'encountered an error while scheduling periodic tasks')
time.sleep(idle_seconds)
def _post_teardown(self):
"""Performs any post-test things. This includes:
* Flushing the contents of the database, to leave a clean slate. If
the class has an 'available_apps' attribute, post_migrate isn't fired.
* Force-closing the connection, so the next test gets a clean cursor.
"""
try:
self._fixture_teardown()
super(TransactionTestCase, self)._post_teardown()
# Some DB cursors include SQL statements as part of cursor
# creation. If you have a test that does rollback, the effect of
# these statements is lost, which can effect the operation of
# tests (e.g., losing a timezone setting causing objects to be
# created with the wrong time). To make sure this doesn't happen,
# get a clean connection at the start of every test.
for conn in connections.all():
conn.close()
finally:
cache.unset_available_apps()
def _post_teardown(self):
"""Performs any post-test things. This includes:
* Putting back the original ROOT_URLCONF if it was changed.
* Force closing the connection, so that the next test gets
a clean cursor.
"""
try:
self._fixture_teardown()
super(TransactionTestCase, self)._post_teardown()
# Some DB cursors include SQL statements as part of cursor
# creation. If you have a test that does rollback, the effect of
# these statements is lost, which can effect the operation of
# tests (e.g., losing a timezone setting causing objects to be
# created with the wrong time). To make sure this doesn't happen,
# get a clean connection at the start of every test.
for conn in connections.all():
conn.close()
finally:
cache.unset_available_apps()
def _nodb_connection(self):
nodb_connection = super()._nodb_connection
try:
nodb_connection.ensure_connection()
except (Database.DatabaseError, WrappedDatabaseError):
warnings.warn(
"Normally Django will use a connection to the 'postgres' database "
"to avoid running initialization queries against the production "
"database when it's not needed (for example, when running tests). "
"Django was unable to create a connection to the 'postgres' database "
"and will use the first PostgreSQL database instead.",
RuntimeWarning
)
for connection in connections.all():
if connection.vendor == 'postgresql' and connection.settings_dict['NAME'] != 'postgres':
return self.__class__(
{**self.settings_dict, 'NAME': connection.settings_dict['NAME']},
alias=self.alias,
)
return nodb_connection
def __call__(self, request):
if not self._config.get("performance_stats"):
return self.get_response(request)
set_request(request)
for conn in connections.all():
wrap_cursor(conn, self._notifier)
metric = RouteMetric(method=request.method)
with metrics.activated_metric(metric):
response = self.get_response(request)
metric.status_code = response.status_code
if "Content-Type" in response:
metric.content_type = response["Content-Type"]
metric.end_time = time.time()
self._notifier.routes.notify(metric)
return response
def afterTest(self, test):
"""
Clean up any changes to the test database.
"""
# Restore transaction support on tests
from django.conf import settings
from django.db import connections
from django.core import mail
from django.core.urlresolvers import clear_url_caches
mail.outbox = []
if hasattr(test.context, '_nosedjango_root_urlconf'):
settings.ROOT_URLCONF = test.context._nosedjango_root_urlconf
clear_url_caches()
use_transaction_isolation = self.should_use_transaction_isolation(
test, settings)
if use_transaction_isolation:
self.restore_transaction_support()
logger.debug("Rolling back")
self.exit_atomics()
self.restore_autocommit()
self.rollback()
if self.transaction_is_managed():
self.transaction.leave_transaction_management()
# If connection is not closed Postgres can go wild with
# character encodings.
for connection in connections.all():
connection.close()
else:
# Have to clear the db even if we're using django because django
# doesn't properly flush the database after a test. It relies on
# flushing before a test, so we want to avoid the case where a
# django test doesn't flush and then a normal test runs, because it
# will expect the db to already be flushed
self._flush_db()
self._loaded_test_fixtures = []
self.call_plugins_method('afterRollback', settings)
