def run_commit_hooks():
"""
Fake transaction commit to run delayed on_commit functions
https://medium.com/gitux/speed-up-django-transaction-hooks-tests-6de4a558ef96
"""
with mock.patch(
"django.db.backends.base.base.BaseDatabaseWrapper.validate_no_atomic_block",
lambda a: False,
):
transaction.get_connection().run_and_clear_commit_hooks()
def run_commit_hooks(testcase):
"""
Fake transaction commit to run delayed on_commit functions
:return:
"""
import mock
for db_name in reversed(testcase._databases_names()):
with mock.patch('django.db.backends.base.base.BaseDatabaseWrapper.validate_no_atomic_block', lambda a: False):
transaction.get_connection(using=db_name).run_and_clear_commit_hooks()
def run_commit_hooks(self):
"""
Fake transaction commit to run delayed on_commit functions
https://medium.com/gitux/speed-up-django-transaction-hooks-tests-6de4a558ef96
"""
for db_name in reversed(self._databases_names()):
with mock.patch(
'django.db.backends.base.base.BaseDatabaseWrapper.'
'validate_no_atomic_block', lambda a: False):
transaction.get_connection(
using=db_name, ).run_and_clear_commit_hooks()
def run_commit_hooks(self):
"""
This is to force on_commit and other commit hooks. Because, in testcases,
those hooks are not run as whole test case is wrapped by outer transaction.
Source: https://medium.com/gitux/speed-up-django-transaction-hooks-tests-6de4a558ef96
"""
for db_name in reversed(self._databases_names()):
with mock.patch(
'django.db.backends.base.base.BaseDatabaseWrapper.validate_no_atomic_block',
lambda a: False):
transaction.get_connection(
using=db_name).run_and_clear_commit_hooks()
def __enter__(self):
connection = transaction.get_connection(self.using)
cache = get_cache(OUTER_ATOMIC_CACHE_NAME)
# By default it is enabled.
enable = True
# If name is set it is only enabled if requested by calling enable_named_outer_atomic().
if self.name:
enable = cache.get(self.name, False)
if enable:
# TestCase setup nests tests in two atomics - one for the test class and one for the individual test.
# The outermost atomic starts a transaction - so does not have a savepoint.
# The inner atomic starts a savepoint around the test.
# So, for tests only, there should be exactly one savepoint_id and two atomic_for_testcase_calls.
# atomic_for_testcase_calls below is added in a monkey-patch for tests only.
if self.ALLOW_NESTED and (self.atomic_for_testcase_calls - len(connection.savepoint_ids)) < 1: # lint-amnesty, pylint: disable=no-member
raise transaction.TransactionManagementError('Cannot be inside an atomic block.')
# Otherwise, this shouldn't be nested in any atomic block.
if not self.ALLOW_NESTED and connection.in_atomic_block:
raise transaction.TransactionManagementError('Cannot be inside an atomic block.')
super().__enter__()
def apply_async(self, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
celery_eager = _get_celery_settings('CELERY_ALWAYS_EAGER')
# New setting to run eager task post transaction
# defaults to `not CELERY_ALWAYS_EAGER`
eager_transaction = _get_celery_settings('CELERY_EAGER_TRANSACTION',
not celery_eager)
if django.VERSION < (1, 6):
if transaction.is_managed() and eager_transaction:
if not transaction.is_dirty():
# Always mark the transaction as dirty
# because we push task in queue that must be fired or discarded
if 'using' in kwargs:
transaction.set_dirty(using=kwargs['using'])
else:
transaction.set_dirty()
_get_task_queue().append((self, args, kwargs))
else:
apply_async_orig = super(PostTransactionTask, self).apply_async
return apply_async_orig(*args, **kwargs)
else:
connection = get_connection()
if connection.in_atomic_block and eager_transaction:
_get_task_queue().append((self, args, kwargs))
else:
return self.original_apply_async(*args, **kwargs)
def test_always_rollback(self, _email_user):
connection = transaction.get_connection()
with patch.object(connection, 'rollback', wraps=connection.rollback) as mock_rollback:
with self.assertRaises(TestException):
confirm_email_change(self.request, self.key)
mock_rollback.assert_called_with()
def __enter__(self, *args, **kwargs):
connection = transaction.get_connection(self.using)
connection._setlistspy_celery_atomic = True
if not hasattr(connection, '_setlistspy_celery_on_commit'):
connection._setlistspy_celery_on_commit = []
connection._setlistspy_celery_on_commit.append([])
super().__enter__(*args, **kwargs)
def table(request, database, table, page, per_page):
# TODO: unit test
conn = get_connection(using=database)
cursor = conn.cursor()
cursor.execute("""SELECT COUNT('id') FROM {table};""".format(table=table))
total_count = cursor.fetchone()[0]
offset = (page - 1) * per_page
to = min(offset + per_page, total_count)
count = to - offset
cursor.execute(
"""SELECT * FROM {table} OFFSET {offset} LIMIT {limit};""".format(
table=table, offset=offset, limit=per_page))
return JsonResponse(
data={
'page':
page,
'total_page':
1 + (total_count // per_page),
'from':
offset + 1,
'to':
to,
'count':
count,
'total_count':
total_count,
'columns': [column.name for column in cursor.description],
'results': [[str(column) for column in record]
for record in cursor.fetchall()]
})
def __enter__(self):
connection = transaction.get_connection(self.using)
if not connection.in_atomic_block:
# Reset state when entering an outermost atomic block.
connection.commit_on_exit = True
connection.needs_rollback = False
if not connection.get_autocommit():
# Pretend we're already in an atomic block to bypass the code
# that disables autocommit to enter a transaction, and make a
# note to deal with this case in __exit__.
connection.in_atomic_block = True
connection.commit_on_exit = False
if connection.in_atomic_block:
# We're already in a transaction; create a savepoint, unless we
# were told not to or we're already waiting for a rollback. The
# second condition avoids creating useless savepoints and prevents
# overwriting needs_rollback until the rollback is performed.
if self.savepoint and not connection.needs_rollback:
sid = connection.savepoint()
connection.savepoint_ids.append(sid)
else:
connection.savepoint_ids.append(None)
else:
if self.immediate:
connection.set_autocommit(False)
connection.cursor().execute('BEGIN IMMEDIATE')
else:
connection.set_autocommit(
False, force_begin_transaction_with_broken_autocommit=True)
connection.in_atomic_block = True
def inner(self, exc_type, exc_value, traceback):
needs_rollback = get_connection(self.using).needs_rollback
try:
original(self, exc_type, exc_value, traceback)
finally:
cachalot_caches.exit_atomic(
self.using, exc_type is None and not needs_rollback)
def _execute_processing_script(self,
base_module,
script_file_path,
sql_params,
sql_renderer=DEFAULT_SQL_RENDERER_METHOD):
conn = transaction.get_connection()
sql = pkgutil.get_data(base_module, script_file_path).decode("utf-8")
for sql_stmt in sql_split(sql):
sql_stmt = sql_stmt.strip()
if sql_stmt:
sql_stmt, params = sql_renderer(sql_stmt, sql_params)
with conn.cursor() as cur:
try:
cur.execute(sql_stmt, params)
except (ProgrammingError, IndexError) as exc:
if isinstance(sql_stmt, bytes):
sql_stmt = sql_stmt.decode("utf-8")
msg = [
f"ERROR in SQL statement: '{exc}'",
f"Script file {os.path.join(base_module.replace('.', os.path.sep), script_file_path)}",
f"STATEMENT: {sql_stmt}",
f"PARAMS: {params}",
f"INPUT_PARAMS: {sql_params}",
]
LOG.error(os.linesep.join(msg))
raise exc
def _inner(*args, **kwargs):
connection = transaction.get_connection(using=using)
with connection.cursor() as c:
c.execute('SET session_replication_role = replica')
ret = func(*args, **kwargs)
c.execute('SET session_replication_role = DEFAULT')
return ret
def __exit__(self, exc_type, exc_value, traceback):
connection = get_connection(self.using)
output_logged_requests = connection.output_logged_requests.pop()
if connection.output_logged_requests:
connection.output_logged_requests[-1] += output_logged_requests
else:
[output_logged_request.create() for output_logged_request in output_logged_requests]
def check_select_for_update(request=None):
if not settings.SELECT_FOR_UPDATE_ENABLED:
return False
atomic_transaction = transaction.get_connection().in_atomic_block
if request:
return request.method not in SAFE_METHODS and atomic_transaction
return atomic_transaction
def _handle_related(self, instance) -> None:
"""
Handle related instances changing by sending a group of tasks,
assuming 'get_instances_from_related' document method always returns list of ids or None
"""
if not DEDConfig.autosync_enabled():
return
sync_group = []
for doc in registry._get_related_doc(instance):
doc_instance = doc(related_instance_to_ignore=instance)
related_model = doc_instance.Django.model
if (not self._is_sync_allowed(related_model) or not isinstance(
instance, doc_instance.Django.related_models)):
continue
related = doc_instance.get_instances_from_related(instance)
if related is None:
continue
task = related_model.get_sync_task()
sync_group += [task.s(obj_id) for obj_id in related]
if not sync_group:
return
sync_group = group(sync_group)
connection = transaction.get_connection()
if not connection.in_atomic_block:
sync_group()
else:
transaction.on_commit(lambda: sync_group())
def __enter__(self):
connection = get_connection(self.using)
if not hasattr(connection, 'transaction_signals_context_list'):
connection.transaction_signals_context_list = []
connection.transaction_signals_context_list.append(TransactionSignalsContext())
def _rebuild_kb_chunk(data):
"""Re-render a chunk of documents.
Note: Don't use host components when making redirects to wiki pages; those
redirects won't be auto-pruned when they're 404s.
"""
log.info('Rebuilding %s documents.' % len(data))
pin_this_thread() # Stick to master.
messages = []
start = time.time()
for pk in data:
message = None
try:
document = Document.objects.get(pk=pk)
# If we know a redirect link to be broken (i.e. if it looks like a
# link to a document but the document isn't there), log an error:
url = document.redirect_url()
if (url and points_to_document_view(url) and
not document.redirect_document()):
log.warn('Invalid redirect document: %d' % pk)
html = document.parse_and_calculate_links()
if document.html != html:
# We are calling update here to so we only update the html
# column instead of all of them. This bypasses post_save
# signal handlers like the one that triggers reindexing.
# See bug 797038 and bug 797352.
Document.objects.filter(pk=pk).update(html=html)
statsd.incr('wiki.rebuild_chunk.change')
else:
statsd.incr('wiki.rebuild_chunk.nochange')
except Document.DoesNotExist:
message = 'Missing document: %d' % pk
except Revision.DoesNotExist:
message = 'Missing revision for document: %d' % pk
except ValidationError as e:
message = 'ValidationError for %d: %s' % (pk, e.messages[0])
except SlugCollision:
message = 'SlugCollision: %d' % pk
except TitleCollision:
message = 'TitleCollision: %d' % pk
if message:
log.debug(message)
messages.append(message)
d = time.time() - start
statsd.timing('wiki.rebuild_chunk', int(round(d * 1000)))
if messages:
subject = ('[%s] Exceptions raised in _rebuild_kb_chunk()' %
settings.PLATFORM_NAME)
mail_admins(subject=subject, message='\n'.join(messages))
if not transaction.get_connection().in_atomic_block:
transaction.commit()
unpin_this_thread() # Not all tasks need to do use the master.
def auto_archive_old_questions():
"""Archive all questions that were created over 180 days ago"""
# Set up logging so it doesn't send Ricky email.
logging.basicConfig(level=logging.ERROR)
# Get a list of ids of questions we're going to go change. We need
# a list of ids so that we can feed it to the update, but then
# also know what we need to update in the index.
days_180 = datetime.now() - timedelta(days=180)
q_ids = list(Question.objects.filter(is_archived=False)
.filter(created__lte=days_180)
.values_list('id', flat=True))
if q_ids:
log.info('Updating %d questions', len(q_ids))
sql = """
UPDATE questions_question
SET is_archived = 1
WHERE id IN (%s)
""" % ','.join(map(str, q_ids))
cursor = connection.cursor()
cursor.execute(sql)
if not transaction.get_connection().in_atomic_block:
transaction.commit()
if settings.ES_LIVE_INDEXING:
try:
# So... the first time this runs, it'll handle 160K
# questions or so which stresses everything. Thus we
# do it in chunks because otherwise this won't work.
#
# After we've done this for the first time, we can nix
# the chunking code.
from kitsune.search.utils import chunked
for chunk in chunked(q_ids, 100):
# Fetch all the documents we need to update.
es_docs = get_documents(QuestionMappingType, chunk)
log.info('Updating %d index documents', len(es_docs))
documents = []
# For each document, update the data and stick it
# back in the index.
for doc in es_docs:
doc[u'question_is_archived'] = True
doc[u'indexed_on'] = int(time.time())
documents.append(doc)
QuestionMappingType.bulk_index(documents)
except ES_EXCEPTIONS:
# Something happened with ES, so let's push index
# updating into an index_task which retries when it
# fails because of ES issues.
index_task.delay(QuestionMappingType, q_ids)
def save(self, **kwargs):
obj = super().save(**kwargs)
fields_changed_dict = self.tracker.changed()
# not sending post_commit signal if fields are not changed
date_updated_only = (len(fields_changed_dict)
== 1) and ('date_updated' in fields_changed_dict)
no_changes = not fields_changed_dict or date_updated_only
if no_changes:
return obj
connection = transaction.get_connection()
if connection.in_atomic_block:
if model_tracker_cache.get(self.cache_key):
# Maintain the oldest values, and adds any new ones.
fields_changed_dict.update(model_tracker_cache[self.cache_key])
model_tracker_cache[self.cache_key] = fields_changed_dict
else:
model_tracker_cache[self.cache_key] = fields_changed_dict
transaction.on_commit(self.post_atomic_commit_handler)
else:
created = 'id' in fields_changed_dict
self.post_commit_hook(fields_changed_dict.keys(), created,
fields_changed_dict)
return obj
def inner(self, exc_type, exc_value, traceback):
needs_rollback = get_connection(self.using).needs_rollback
try:
original(self, exc_type, exc_value, traceback)
finally:
cachalot_caches.exit_atomic(
self.using, exc_type is None and not needs_rollback)
def apply_async(self, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
if django.VERSION < (1, 6):
if transaction.is_managed() and not current_app.conf.CELERY_ALWAYS_EAGER:
if not transaction.is_dirty():
# Always mark the transaction as dirty
# because we push task in queue that must be fired or discarded
if 'using' in kwargs:
transaction.set_dirty(using=kwargs['using'])
else:
transaction.set_dirty()
_get_task_queue().append((self, args, kwargs))
else:
apply_async_orig = super(PostTransactionTask, self).apply_async
return apply_async_orig(*args, **kwargs)
else:
connection = get_connection()
if connection.in_atomic_block and not getattr(current_app.conf, 'CELERY_ALWAYS_EAGER', False):
_get_task_queue().append((self, args, kwargs))
else:
return self.original_apply_async(*args, **kwargs)
def _transactions_mark_order_dirty(order_id, using=None):
if "PYTEST_CURRENT_TEST" in os.environ:
# We don't care about Order.objects.create() calls in test code so let's try to figure out if this is test code
# or not.
for frame in inspect.stack():
if 'pretix/base/models/orders' in frame.filename:
continue
elif 'test_' in frame.filename or 'conftest.py in frame.filename':
return
elif 'pretix/' in frame.filename or 'pretix_' in frame.filename:
# This went through non-test code, let's consider it non-test
break
if order_id is None:
return
conn = transaction.get_connection(using)
if not conn.in_atomic_block:
_fail(
"You modified an Order, OrderPosition, or OrderFee object in a way that should create "
"a new Transaction object within the same database transaction, however you are not "
"doing it inside a database transaction!")
if getattr(dirty_transactions, 'order_ids', None) is None:
dirty_transactions.order_ids = set()
if _check_for_dirty_orders not in [
func for savepoint_id, func in conn.run_on_commit
]:
transaction.on_commit(_check_for_dirty_orders, using)
dirty_transactions.order_ids.clear(
) # This is necessary to clean up after old threads with rollbacked transactions
dirty_transactions.order_ids.add(order_id)
def import_communes_test_data():
with get_connection().cursor() as cursor:
cursor.execute(
"ALTER TABLE data_france_commune DROP CONSTRAINT IF EXISTS commune_departement_constraint;"
)
with (DATA_DIR / "commune.csv").open() as f:
cursor.copy_from(
f,
"data_france_commune",
columns=[
"id",
"code",
"type",
"nom",
"type_nom",
"population_municipale",
"population_cap",
"geometry",
"search",
"commune_parent_id",
],
)
with (DATA_DIR / "codepostal.csv").open() as f:
cursor.copy_from(
f,
"data_france_codepostal",
columns=["id", "code"],
)
with (DATA_DIR / "codepostal_communes.csv").open() as f:
cursor.copy_from(
f,
"data_france_codepostal_communes",
columns=["id", "codepostal_id", "commune_id"],
)
def prepare_transaction(self):
# django DB connection 은 thread safe 함
connection = get_connection(self.using)
self.connection = connection
outermost_transaction = getattr(connection, "outermost_transaction",
None)
is_self_outermost_transaction = outermost_transaction is None
if is_self_outermost_transaction:
assert getattr(connection, "tran_stack", []) == []
self.is_outermost = True
connection.outermost_transaction = self
connection.tran_stack = [self]
self.key_gen = self.create_key_generator()
self.id = self.key_gen()
checkin_actor, login_user = self.checkin_actor, self.login_user
if checkin_actor:
assert checkin_actor.status in (Status.NORMAL, Status.NEW)
self.set(checkin_actor)
if login_user:
assert login_user.status in (Status.NORMAL, Status.NEW)
self.set(login_user)
else:
self.is_outermost = False
tran_stack = connection.tran_stack
parent = tran_stack[-1]
if not self.is_readonly:
assert not parent.is_readonly, "ReadonlyTransaction 안에는 ReadonlyTransaction 만 들어올 수 있습니다"
tran_stack.append(self)
self.key_gen = outermost_transaction.key_gen
self.id = outermost_transaction.id
if not self._checkin_actor:
self._checkin_actor = parent.checkin_actor
if not self._login_user:
self._login_user = parent._login_user
self.connect_storage(outermost_transaction)
def get_tenant(self, model, hostname, request):
"""Override the tenant selection logic."""
connections["default"].set_schema_to_public()
tenant_schema = create_schema_name(request.user.account)
tenant = TENANTS.get_tenant(tenant_schema)
if tenant is None:
if request.user.system:
try:
tenant = Tenant.objects.get(schema_name=tenant_schema)
except Tenant.DoesNotExist:
raise Http404()
else:
with transaction.atomic():
try:
tenant = Tenant.objects.get(schema_name=tenant_schema)
except Tenant.DoesNotExist:
cursor = transaction.get_connection().cursor()
cursor.execute("LOCK TABLE public.api_tenant in SHARE ROW EXCLUSIVE MODE")
tenant, created = Tenant.objects.get_or_create(schema_name=tenant_schema)
if created:
seed_permissions(tenant=tenant)
seed_roles(tenant=tenant)
seed_group(tenant=tenant)
TENANTS.save_tenant(tenant)
return tenant
def test_always_rollback(self, _email_user):
connection = transaction.get_connection()
with patch.object(connection, 'rollback', wraps=connection.rollback) as mock_rollback:
with self.assertRaises(TestException):
confirm_email_change(self.request, self.key)
mock_rollback.assert_called_with()
def check_select_for_update(request=None):
if not settings.SELECT_FOR_UPDATE_ENABLED:
return False
atomic_transaction = transaction.get_connection().in_atomic_block
if request:
return request.method not in SAFE_METHODS and atomic_transaction
return atomic_transaction
def apply_async(self, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
celery_eager = _get_celery_settings('CELERY_ALWAYS_EAGER')
# New setting to run eager task post transaction
# defaults to `not CELERY_ALWAYS_EAGER`
eager_transaction = _get_celery_settings('CELERY_EAGER_TRANSACTION',
not celery_eager)
if django.VERSION < (1, 6):
if transaction.is_managed() and eager_transaction:
if not transaction.is_dirty():
# Always mark the transaction as dirty
# because we push task in queue that must be fired or discarded
if 'using' in kwargs:
transaction.set_dirty(using=kwargs['using'])
else:
transaction.set_dirty()
_get_task_queue().append((self, args, kwargs))
else:
apply_async_orig = super(PostTransactionTask, self).apply_async
return apply_async_orig(*args, **kwargs)
else:
connection = get_connection()
if connection.in_atomic_block and eager_transaction:
_get_task_queue().append((self, args, kwargs))
else:
return self.original_apply_async(*args, **kwargs)
def handle(self, **options):
# Set up logging so it doesn't send Ricky email.
logging.basicConfig(level=logging.ERROR)
# Get a list of ids of questions we're going to go change. We need
# a list of ids so that we can feed it to the update, but then
# also know what we need to update in the index.
days_180 = datetime.now() - timedelta(days=180)
q_ids = list(
Question.objects.filter(is_archived=False).filter(
created__lte=days_180).values_list("id", flat=True))
if q_ids:
log.info("Updating %d questions", len(q_ids))
sql = """
UPDATE questions_question
SET is_archived = 1
WHERE id IN (%s)
""" % ",".join(map(str, q_ids))
cursor = connection.cursor()
cursor.execute(sql)
if not transaction.get_connection().in_atomic_block:
transaction.commit()
if settings.ES_LIVE_INDEXING:
# elastic v7 code:
answer_ids = list(
Answer.objects.filter(question_id__in=q_ids).values_list(
"id", flat=True))
index_objects_bulk.delay("QuestionDocument", q_ids)
index_objects_bulk.delay("AnswerDocument", answer_ids)
def __enter__(self):
if not self.ENABLED:
return
connection = transaction.get_connection(self.using)
if connection.in_atomic_block:
raise transaction.TransactionManagementError('Cannot be inside an atomic block.')
if getattr(connection, 'commit_on_success_block_level', 0) == 0:
connection.commit_on_success_block_level = 1
# This will set the transaction isolation level to READ COMMITTED for the next transaction.
if self.read_committed is True:
if connection.vendor == 'mysql':
cursor = connection.cursor()
cursor.execute("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
# We aren't in a transaction yet; create one.
# The usual way to start a transaction is to turn autocommit off.
# However, some database adapters (namely sqlite3) don't handle
# transactions and savepoints properly when autocommit is off.
# In such cases, start an explicit transaction instead, which has
# the side-effect of disabling autocommit.
if connection.features.autocommits_when_autocommit_is_off:
connection._start_transaction_under_autocommit() # pylint: disable=protected-access
connection.autocommit = False
else:
connection.set_autocommit(False)
else:
if self.read_committed is True:
raise transaction.TransactionManagementError('Cannot change isolation level when nested.')
connection.commit_on_success_block_level += 1
def transaction_committed(self):
conn = transaction.get_connection()
if conn.in_atomic_block:
# committed nested transaction - ensure hook is attached
self._transaction_savepts = conn.savepoint_ids
conn.on_commit(self.transaction_committed)
else:
for using, instances in self._transaction_removed.items():
if instances:
LOGGER.debug(
"Committing %d deferred Solr delete(s) after transaction",
len(instances))
if self._backend_queue:
self._backend_queue.delete(self.__class__, using,
list(instances.values()))
else:
backend = self.get_backend(using)
if backend is not None:
for instance in instances.values():
backend.remove(instance)
for using, instances in self._transaction_added.items():
if instances:
LOGGER.debug(
"Committing %d deferred Solr update(s) after transaction",
len(instances))
if self._backend_queue:
self._backend_queue.add(self.__class__, using,
list(instances.values()))
else:
backend = self.get_backend(using)
if backend is not None:
backend.update(self, instances.values())
self.reset()
def __enter__(self):
connection = get_connection(self.using)
if not connection.in_atomic_block:
connection.begin_immediate = True
self.set_begin_immediate = True
super().__enter__()
def save(self, *args, **kwargs):
# Wrap around a transaction only if we're not already in a transaction.
connection = transaction.get_connection()
if not connection.in_atomic_block:
with transaction.atomic():
return super().save(*args, **kwargs)
return super().save(*args, **kwargs)
def test_reprepare_query_in_transaction(self):
"""
Given an ORM query is prepared
And a database transaction is started
And the query has been deallocated in the database
When the query is executed
Then it should re-prepare without errors
And it should execute as expected
"""
def all_species():
return Species.prepare.all()
PreparedStatementController().register_qs("all_species", all_species)
PreparedStatementController().prepare_qs_stmt("all_species",
force=True)
# deallocate the query to simulate changing database session
PreparedStatementController(
).prepared_statements["all_species"].deallocate()
with transaction.atomic():
qs = execute_stmt("all_species")
cxn = transaction.get_connection()
self.assertTrue(cxn.in_atomic_block)
self.assertEqual(len(qs), 3)
def _send_tasks(**kwargs):
"""Sends all delayed Celery tasks.
Called after a transaction is committed or we leave a transaction
management block in which no changes were made (effectively a commit).
"""
# Detect test mode through CELERY_ALWAYS_EAGER settings
# We assume all celery transactions tests on 1.8+ are running with TestCase, otherwise we'd get atomic exceptions
celery_eager = _get_celery_settings('CELERY_ALWAYS_EAGER')
# If we detect higher up nested atomic block, continue
connection = get_connection()
if django.VERSION >= (1, 8):
min_number_transactions = 1 if celery_eager else 0
if (not celery_eager and connection.in_atomic_block) or len(
connection.savepoint_ids) > min_number_transactions:
return
elif connection.in_atomic_block:
return
queue = _get_task_queue()
while queue:
tsk, args, kwargs = queue.pop(0)
tsk.original_apply_async(*args, **kwargs)
def __enter__(self):
connection = transaction.get_connection(self.using)
# TestCase setup nests tests in two atomics - one for the test class and one for the individual test.
# The outermost atomic starts a transaction - so does not have a savepoint.
# The inner atomic starts a savepoint around the test.
# So, for tests only, there should be exactly one savepoint_id and two atomic_for_testcase_calls.
# atomic_for_testcase_calls below is added in a monkey-patch for tests only.
if self.ALLOW_NESTED and (self.atomic_for_testcase_calls -
len(connection.savepoint_ids)) < 1:
raise transaction.TransactionManagementError(
'Cannot be inside an atomic block.')
# Otherwise, this shouldn't be nested in any atomic block.
if not self.ALLOW_NESTED and connection.in_atomic_block:
raise transaction.TransactionManagementError(
'Cannot be inside an atomic block.')
# This will set the transaction isolation level to READ COMMITTED for the next transaction.
if self.read_committed is True:
if connection.vendor == 'mysql':
cursor = connection.cursor()
cursor.execute(
"SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
super(OuterAtomic, self).__enter__()
def backup_instance(self, storage, instance, query_args=None):
""" query_args is a dict in a form of
{
'model_backup_name': [id_list],
'model_backup_name3': [], #no data
}
If there is no key in query_args, queryset is not filtered
"""
with instance_context(instance):
db = get_instance_db(instance)
# get migrations
targets = self.get_instance_migrations(instance)
storage.start_model(self.MIGRATIONS_STORAGE)
for target in targets:
storage.append(target)
storage.end_model()
with transaction.atomic(using=db):
cursor = transaction.get_connection(db).cursor()
cursor.execute(
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ;')
for model in self.default_sorted:
options = self.get_options_for_model(model)
storage.start_model(options.get_name())
try:
options.backup(storage, query_args)
except Exception:
logger.warning('Exception for model %s',
model,
exc_info=1)
raise
storage.end_model()
def get_df(lname):
conn = get_connection('datastore')
gc_df = pd.read_sql('select * from geometry_columns',
conn,
index_col='f_table_name')
srid = gc_df.loc[lname].srid
gtype = gc_df.loc[lname].type
gname = gc_df.loc[lname].f_geometry_column
if gtype in ('POLYGON', 'MULTIPOLYGON'):
sql = "select *, st_buffer({}, 0) as the_geom_clean from {} where {} is not null".format(
gname, lname, gname)
else:
sql = "select *, {} as the_geom_clean from {} where {} is not null".format(
gname, lname, gname)
# print sql
gdf = gpd.GeoDataFrame.from_postgis(sql,
conn,
geom_col='the_geom_clean',
crs={
'init': 'epsg:{}'.format(srid),
'no_defs': True
})
# print lname, srid
gdf.to_crs(epsg="3035", inplace=True)
return gdf
def __enter__(self):
connection = transaction.get_connection(self.using)
cache = get_cache(OUTER_ATOMIC_CACHE_NAME)
# By default it is enabled.
enable = True
# If name is set it is only enabled if requested by calling enable_named_outer_atomic().
if self.name:
enable = cache.get(self.name, False)
if enable:
# TestCase setup nests tests in two atomics - one for the test class and one for the individual test.
# The outermost atomic starts a transaction - so does not have a savepoint.
# The inner atomic starts a savepoint around the test.
# So, for tests only, there should be exactly one savepoint_id and two atomic_for_testcase_calls.
# atomic_for_testcase_calls below is added in a monkey-patch for tests only.
if self.ALLOW_NESTED and (self.atomic_for_testcase_calls - len(connection.savepoint_ids)) < 1:
raise transaction.TransactionManagementError('Cannot be inside an atomic block.')
# Otherwise, this shouldn't be nested in any atomic block.
if not self.ALLOW_NESTED and connection.in_atomic_block:
raise transaction.TransactionManagementError('Cannot be inside an atomic block.')
# This will set the transaction isolation level to READ COMMITTED for the next transaction.
if self.read_committed is True:
if connection.vendor == 'mysql':
cursor = connection.cursor()
cursor.execute("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
super(OuterAtomic, self).__enter__()
def apply_async(cls, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
connection = get_connection()
if connection.in_atomic_block and not getattr(current_app.conf, 'CELERY_ALWAYS_EAGER', False):
_get_task_queue().append((cls, args, kwargs))
else:
return cls.original_apply_async(*args, **kwargs)
def __exit__(self, exc_type, exc_value, traceback):
self._no_monkey.__exit__(self, exc_type, exc_value, traceback)
connection = get_connection(self.using)
if not connection.closed_in_transaction and exc_type is None and \
not connection.needs_rollback:
transaction_state.commit()
else:
transaction_state.rollback()
def __exit__(self, exc_type, exc_value, traceback):
connection = get_connection(self.using)
transaction_signals_context = connection.transaction_signals_context_list.pop()
if not exc_value:
if len(connection.transaction_signals_context_list) == 0:
transaction_signals_context.handle_all()
else:
connection.transaction_signals_context_list[-1].join(transaction_signals_context)
def commit_on_success(using=None): # noqa
connection = transaction.get_connection(using)
if connection.features.autocommits_when_autocommit_is_off:
# ignore stupid warnings and errors
yield
else:
with transaction.atomic(using):
yield
def _on_exit_without_update(self, using):
'''
Clear signals on transaction exit, even if neither commit nor rollback
happened.
'''
if self._has_signals():
connection = get_connection(using)
self._remove_signals(connection.savepoint_ids)
def update_question_vote_chunk(data):
"""Update num_votes_past_week for a number of questions."""
# First we recalculate num_votes_past_week in the db.
log.info('Calculating past week votes for %s questions.' % len(data))
ids = ','.join(map(str, data))
sql = """
UPDATE questions_question q
SET num_votes_past_week = (
SELECT COUNT(created)
FROM questions_questionvote qv
WHERE qv.question_id = q.id
AND qv.created >= DATE(SUBDATE(NOW(), 7))
)
WHERE q.id IN (%s);
""" % ids
cursor = connection.cursor()
cursor.execute(sql)
if not transaction.get_connection().in_atomic_block:
transaction.commit()
# Next we update our index with the changes we made directly in
# the db.
if data and settings.ES_LIVE_INDEXING:
# Get the data we just updated from the database.
sql = """
SELECT id, num_votes_past_week
FROM questions_question
WHERE id in (%s);
""" % ids
cursor = connection.cursor()
cursor.execute(sql)
# Since this returns (id, num_votes_past_week) tuples, we can
# convert that directly to a dict.
id_to_num = dict(cursor.fetchall())
try:
# Fetch all the documents we need to update.
from kitsune.questions.models import QuestionMappingType
from kitsune.search import es_utils
es_docs = es_utils.get_documents(QuestionMappingType, data)
# For each document, update the data and stick it back in the
# index.
for doc in es_docs:
# Note: Need to keep this in sync with
# Question.extract_document.
num = id_to_num[int(doc[u'id'])]
doc[u'question_num_votes_past_week'] = num
QuestionMappingType.index(doc, id_=doc['id'])
except ES_EXCEPTIONS:
# Something happened with ES, so let's push index updating
# into an index_task which retries when it fails because
# of ES issues.
index_task.delay(QuestionMappingType, id_to_num.keys())
def __patched__exit__(self, exc_type, exc_value, trackback):
connection = get_connection(self.using)
if connection.savepoint_ids:
sid = connection.savepoint_ids.pop()
else:
# Prematurely unset this flag to allow using commit or rollback.
connection.in_atomic_block = False
try:
if exc_type is None and not connection.needs_rollback:
if connection.in_atomic_block:
# Release savepoint if there is one
if sid is not None:
try:
connection.savepoint_commit(sid)
transaction.signals.post_commit.send(None)
except DatabaseError:
connection.savepoint_rollback(sid)
transaction.signals.post_rollback.send(None)
raise
else:
# Commit transaction
try:
connection.commit()
transaction.signals.post_commit.send(None)
except DatabaseError:
connection.rollback()
transaction.signals.post_rollback.send(None)
raise
else:
# This flag will be set to True again if there isn't a savepoint
# allowing to perform the rollback at this level.
connection.needs_rollback = False
if connection.in_atomic_block:
# Roll back to savepoint if there is one, mark for rollback
# otherwise.
if sid is None:
connection.needs_rollback = True
else:
connection.savepoint_rollback(sid)
transaction.signals.post_rollback.send(None)
else:
# Roll back transaction
connection.rollback()
transaction.signals.post_rollback.send(None)
finally:
# Outermost block exit when autocommit was enabled.
if not connection.in_atomic_block:
if connection.features.autocommits_when_autocommit_is_off:
connection.autocommit = True
else:
connection.set_autocommit(True)
# Outermost block exit when autocommit was disabled.
elif not connection.savepoint_ids and not connection.commit_on_exit:
connection.in_atomic_block = False
def __enter__(self):
assert self.cxn is None, "lock already acquired"
self.cxn = transaction.get_connection(self.using)
if not self.cxn.in_atomic_block:
self.txn = transaction.atomic(using=self.using)
self.txn.__enter__()
self.cur = self.cxn.cursor()
self.cur.execute("LOCK TABLE %s IN %s MODE" %(
self.model._meta.db_table, self.mode,
))
def test_mark_for_rollback_on_error_in_autocommit(self):
self.assertTrue(transaction.get_autocommit())
# Swallow the intentional error raised.
with self.assertRaisesMessage(Exception, "Oops"):
# Wrap in `mark_for_rollback_on_error` to check if the transaction is marked broken.
with transaction.mark_for_rollback_on_error():
# Ensure that we are still in a good state.
self.assertFalse(transaction.get_connection().needs_rollback)
raise Exception("Oops")
# Ensure that `mark_for_rollback_on_error` did not mark the transaction
# as broken, since we are in autocommit mode …
self.assertFalse(transaction.get_connection().needs_rollback)
# … and further queries work nicely.
Reporter.objects.create()
def __enter__(self):
"""Context manager for locking table."""
super().__enter__()
connection = get_connection(self.using)
with connection.cursor() as cursor:
cursor.execute(
'LOCK TABLE {db_table_name} IN {lock_mode} MODE'.format(
db_table_name=self.model._meta.db_table,
lock_mode=self.lock_mode
)
)
def log_output_request(data, related_objects=None, using=None):
"""
Helper for logging output requests
:param data: dict of input attributes of OutputLoggedRequest model
:param related_objects: objects that will be related to OutputLoggedRequest object
:param using: database alias
"""
if is_active_logged_requests(using):
output_logged_requests = get_connection(using).output_logged_requests[-1]
output_logged_requests.append(OutputLoggedRequestContext(data, related_objects))
else:
output_logged_request = OutputLoggedRequest.objects.create(**data)
if related_objects:
[output_logged_request.related_objects.create(content_object=obj) for obj in related_objects]
def schedule_changed(self):
try:
# If MySQL is running with transaction isolation level
# REPEATABLE-READ (default), then we won't see changes done by
# other transactions until the current transaction is
# committed (Issue #41).
try:
transaction.commit()
except transaction.TransactionManagementError:
pass # not in transaction management.
last, ts = self._last_timestamp, self.Changes.last_change()
except DATABASE_ERRORS as exc:
# as per https://github.com/celery/django-celery/commit/f2ad0848e3f55ee9a4c5d3ffeff62f4d54036ed7
transaction.get_connection().close_if_unusable_or_obsolete()
error('Database gave error: %r', exc, exc_info=1)
return False
try:
if ts and ts > (last if last else ts):
return True
finally:
self._last_timestamp = ts
return False
def process_view(self, request, view_func, view_args, view_kwargs):
# This uses undocumented django APIS:
# - transaction.get_connection() followed by in_atomic_block property,
# which we need to make sure we're not messing with a transaction
# that has already started (which happens in tests using the regular
# TestCase class)
# - _non_atomic_requests(), which set the property to prevent the
# transaction on the view itself. We can't use non_atomic_requests
# (without the '_') as it returns a *new* view, and we can't do that
# in a middleware, we need to modify it in place and return None so
# that the rest of the middlewares are run.
is_method_safe = request.method in ('HEAD', 'GET', 'OPTIONS', 'TRACE')
if is_method_safe and not transaction.get_connection().in_atomic_block:
transaction._non_atomic_requests(view_func, using='default')
return None
def __enter__(self):
super(LockedAtomicTransaction, self).__enter__()
# Make sure not to lock, when sqlite is used, or you'll run into problems while running tests!!!
if settings.DATABASES[self.using]["ENGINE"] != "django.db.backends.sqlite3":
cursor = None
try:
cursor = get_connection(self.using).cursor()
for model in self.models:
cursor.execute(
"LOCK TABLE {table_name}".format(table_name=model._meta.db_table)
)
finally:
if cursor and not cursor.closed:
cursor.close()
def pg_version(using=None):
"""
Return tuple with PostgreSQL version of a specific connection
:type using: str
:param using: Connection name
:rtype: tuple
:return: PostgreSQL version
"""
connection = get_connection(using)
cursor = connection.cursor()
cursor.execute('SHOW server_version')
row = cursor.fetchone()
return tuple([int(i) for i in row[0].split('.')])
def apply_async(self, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
celery_eager = _get_celery_settings("CELERY_ALWAYS_EAGER")
# New setting to run eager task post transaction
# defaults to `not CELERY_ALWAYS_EAGER`
eager_transaction = _get_celery_settings("CELERY_EAGER_TRANSACTION", not celery_eager)
connection = get_connection()
if connection.in_atomic_block and eager_transaction:
_get_task_queue().append((self, args, kwargs))
else:
return self.original_apply_async(*args, **kwargs)
def apply_async(self, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
using = kwargs['using'] if 'using' in kwargs else None
con = transaction.get_connection(using)
if con.get_autocommit() or con.in_atomic_block:
if not transaction.is_dirty():
# Always mark the transaction as dirty
# because we push task in queue that must be fired or discarded
transaction.set_dirty(using=using)
task = lambda: self.original_apply_async(*args, **kwargs)
connection.on_commit(task)
else:
return self.original_apply_async(*args, **kwargs)
def defer(f, *args, **kwargs):
'''
Wrapper that defers a function's execution until the current transaction
commits, if a transaction is active. Otherwise, executes as usual. Note
that a deferred function will NOT be called if the transaction completes
without committing (e.g. when transaction.is_dirty() is False upon exiting
the transaction).
An implicit assumption is that a deferred function does not return an
important value, since there is no way to retrieve the return value in
the normal execution order.
Before being connected to the 'post_commit' signal of an existing managed
transaction, the deferred function is wrapped by the @commit_on_success
decorator to ensure that it behaves properly by committing or rolling back
any updates it makes to a current transaction.
>>> from django.db import transaction
>>> from django_transaction_signals import defer
>>>
>>> def log_success(msg):
>>> print 'logging success'
>>>
>>> @transaction.atomic
>>> def transactional_update(value):
>>> print 'starting transaction'
>>> defer(log_success, 'The transaction was successful')
>>> print 'finishing transaction'
>>>
>>> transactional_update('foo')
... starting transaction
... finishing transaction
... logging success
'''
connection = get_connection(kwargs.pop('using', None))
if not connection.get_autocommit() or connection.in_atomic_block:
def f_deferred(*a, **kw):
f(*args, **kwargs)
if connection.savepoint_ids:
savepoint_id = connection.savepoint_ids[-1]
else:
savepoint_id = None
dispatch_uid = (savepoint_id, _make_id(f_deferred))
transaction.signals.post_commit.connect(f_deferred, weak=False, dispatch_uid=dispatch_uid)
else:
return f(*args, **kwargs)
def on_success(handler, using=None):
"""
Register a handler or a function to be called after successful code pass.
If transaction signals are not active the handler/function is called immediately.
:param handler: handler or function that will be called.
:param using: name of the database
"""
connection = get_connection(using)
if getattr(connection, 'transaction_signals_context_list', False):
connection.transaction_signals_context_list[-1].register(handler)
else:
if settings.DEBUG:
logger.warning(
'For on success signal should be activated transaction signals via transaction_signals decorator.'
'Function is called immediately now.'
)
handler()
