def runFilters(self, filters, dryRun=False):
using = DEFAULT_DB_ALIAS
transaction.enter_transaction_management(using=using)
try:
for datafile in DataFile.objects.all():
# Use a transaction to process each Datafile
transaction.managed(True, using=using)
try:
for filter in filters:
filter(sender=DataFile,
instance=datafile,
created=False,
using='default')
if dryRun:
transaction.rollback(using=using)
else:
transaction.commit(using=using)
except Exception:
transaction.rollback(using=using)
exc_class, exc, tb = sys.exc_info()
new_exc = CommandError("Exception %s has occurred: "
"rolled back transaction" % \
(exc or exc_class))
raise new_exc.__class__, new_exc, tb
finally:
transaction.leave_transaction_management(using=using)
def grab_db_lock(lock_name, wait):
"""
Grab a lock using a new, temporary connection. Yields a "success"
boolean indicating whether the lock was successfully acquired or not.
This context manager ensures that multidb does not override the
connection when accessing the Lock model and its transaction. The
code that runs within this context should NOT be affected - it must
run as usual, with normal multidb functionality intact.
"""
with connections.get() as using:
try:
with connection_state.force(None):
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
lock = Lock.grab(lock_name, wait=wait, using=using)
success = bool(lock)
try:
yield success
finally:
if success:
with connection_state.force(None):
lock.release(using=using)
transaction.commit(using=using)
finally:
with connection_state.force(None):
transaction.leave_transaction_management(using=using)
def execute_transaction(sql, output=False, database='default'):
"A transaction wrapper for executing a list of SQL statements"
my_connection = connection
using_args = {}
if is_multi_db():
if not database:
database = DEFAULT_DB_ALIAS
my_connection = connections[database]
using_args['using'] = database
try:
# Begin Transaction
transaction.enter_transaction_management(**using_args)
transaction.managed(True, **using_args)
cursor = my_connection.cursor()
# Perform the SQL
if output:
write_sql(sql, database)
execute_sql(cursor, sql)
transaction.commit(**using_args)
transaction.leave_transaction_management(**using_args)
except Exception:
transaction.rollback(**using_args)
raise
def run_select_for_update(self, status, nowait=False):
"""
Utility method that runs a SELECT FOR UPDATE against all
Person instances. After the select_for_update, it attempts
to update the name of the only record, save, and commit.
This function expects to run in a separate thread.
"""
status.append('started')
try:
# We need to enter transaction management again, as this is done on
# per-thread basis
transaction.enter_transaction_management()
transaction.managed(True)
people = list(
Person.objects.all().select_for_update(nowait=nowait)
)
people[0].name = 'Fred'
people[0].save()
transaction.commit()
except DatabaseError as e:
status.append(e)
finally:
# This method is run in a separate thread. It uses its own
# database connection. Close it without waiting for the GC.
transaction.abort()
connection.close()
def setUp(self):
super(TransactionalTestCase, self).setUp()
transaction.enter_transaction_management()
transaction.managed(True)
self.setUpInTransaction()
def beforeTest(self, test):
if not self.settings_path:
# short circuit if no settings file can be found
return
from django.core.management import call_command
from django.core.urlresolvers import clear_url_caches
from django.db import connection, transaction
from django.core import mail
mail.outbox = []
transaction_support = self._has_transaction_support(test)
if transaction_support:
transaction.enter_transaction_management()
transaction.managed(True)
self.disable_transaction_support(transaction)
if isinstance(test, nose.case.Test) and \
isinstance(test.test, nose.case.MethodTestCase) and \
hasattr(test.context, 'fixtures'):
# We have to use this slightly awkward syntax due to the fact
# that we're using *args and **kwargs together.
call_command('loaddata', *test.context.fixtures, **{'verbosity': 0})
if isinstance(test, nose.case.Test) and \
isinstance(test.test, nose.case.MethodTestCase) and \
hasattr(test.context, 'urls'):
# We have to use this slightly awkward syntax due to the fact
# that we're using *args and **kwargs together.
self.old_urlconf = settings.ROOT_URLCONF
settings.ROOT_URLCONF = self.urls
clear_url_caches()
def handle(self, *args, **options):
print("Download zip-archive...")
f = urlopen(IPGEOBASE_SOURCE_URL)
buffer = BytesIO(f.read())
f.close()
print("Unpacking...")
zip_file = ZipFile(buffer)
cities_file_read = _read_file(zip_file, 'cities.txt')
cidr_optim_file_read = _read_file(zip_file, 'cidr_optim.txt')
zip_file.close()
buffer.close()
print("Start updating...")
list_cities = cities_file_read.decode(IPGEOBASE_CODING).split('\n')
list_cidr_optim = \
cidr_optim_file_read.decode(IPGEOBASE_CODING).split('\n')
lines = \
_get_cidr_optim_with_cities_lines(list_cidr_optim, list_cities)
cursor = connection.cursor()
transaction.enter_transaction_management()
try:
transaction.managed(True)
print("Delete old rows in table ipgeobase...")
cursor.execute(DELETE_SQL)
print ("Write new data...")
cursor.executemany(INSERT_SQL, [l for l in lines if l])
transaction.commit()
except Exception as e:
message = "The data not updated:", e
if send_message:
mail_admins(subject=ERROR_SUBJECT, message=message)
raise CommandError(message)
finally:
transaction.rollback()
transaction.leave_transaction_management()
return "Table ipgeobase is update.\n"
def handle(self, **options):
'''
Sample data:
??crawl_id;start_time;end_time;sucess;error_if_any;day;month;year;dayofweek;datestring
1;2009-01-07 14:00:00;2009-01-07 14:02:51;YES;NO;7;1;2009;4;2009.01.07
2;2009-01-07 15:00:00;2009-01-07 15:02:27;YES;NO;7;1;2009;4;2009.01.07
3;2009-01-07 16:00:01;2009-01-07 16:02:17;YES;NO;7;1;2009;4;2009.01.07
4;2009-01-07 17:00:00;2009-01-07 17:02:49;YES;NO;7;1;2009;4;2009.01.07
model:
start_time = models.DateTimeField('Start Time')
end_time = models.DateTimeField('End Time')
success = models.BooleanField('Successfoul crawl?')
groups_downloaded = models.IntegerField('Groups downloaded')
errors = JSONField('Errors', blank=True, null=True)
old_id = models.IntegerField(null=True,blank=True)
'''
try:
f = open(
os.path.join(settings.ROOT_PATH, 'data', 'crawl.utf8.csv'),
"rb")
f.readline()
transaction.enter_transaction_management()
transaction.managed(True)
i = 0
for row in f:
row = unicode(row)
row = row.strip()
row = row.split(';')
if len(row) == 10:
obj = Crawl(
**{
'old_id':
int(row[0]),
'start_time':
datetime.datetime.strptime(row[1],
"%Y-%m-%d %H:%M:%S"),
'end_time':
datetime.datetime.strptime(row[2],
"%Y-%m-%d %H:%M:%S"),
'success':
True if row[3] == 'YES' else False,
'groups_downloaded':
0
})
#obj.save()
i = i + 1
if row[3] == 'NO': print row[0]
#transaction.commit()
print 'saved %s rows' % i
except KeyboardInterrupt:
transaction.rollback()
exit()
def handle(self, **options):
'''The main entry point for the Django management command.'''
import_start = datetime.datetime.now()
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
try:
self._promote_devel()
except:
self._rollback_db()
raise
# Finalize the transaction and close the db connection.
transaction.commit()
transaction.leave_transaction_management()
connection.close()
import_end = datetime.datetime.now()
# Print a short summary of what we did.
td = import_end - import_start
print '\nProcessing complete in %s days, %s.%s seconds.' % (
td.days, td.seconds, td.microseconds)
print ' TraitData objects promoted: %s' % (
PublicTraitData.objects.all().count(), )
def handle(self, quiet=False, *files, **options):
connection = connections[DEFAULT_DB_ALIAS]
self.style = no_style()
cursor = connection.cursor()
# we do all messages in one transaction
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
# our current date/time in UTC
utc_time = datetime.now(pytz.utc)
try:
# check our reports
sent = Report.check_all(utc_time, quiet)
if not quiet:
print
print "%d reports sent." % sent
except Exception as e:
traceback.print_exc(e)
# one uber commit
transaction.commit()
def run_select_for_update(self, status, nowait=False):
"""
Utility method that runs a SELECT FOR UPDATE against all
Person instances. After the select_for_update, it attempts
to update the name of the only record, save, and commit.
This function expects to run in a separate thread.
"""
status.append('started')
try:
# We need to enter transaction management again, as this is done on
# per-thread basis
transaction.enter_transaction_management(True)
transaction.managed(True)
people = list(
Person.objects.all().select_for_update(nowait=nowait))
people[0].name = 'Fred'
people[0].save()
transaction.commit()
except DatabaseError as e:
status.append(e)
finally:
# This method is run in a separate thread. It uses its own
# database connection. Close it without waiting for the GC.
connection.close()
def forwards(self, orm):
"Write your forwards migration here"
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
start = time.time()
map = {}
lc = {}
for tc in ThreadedComment.objects.filter(
tree_path='').order_by('id').iterator():
path_leaf = unicode(tc.pk).zfill(PATH_DIGITS)
if tc.parent_id:
parent_path = map[tc.parent_id]
tree_path = PATH_SEPARATOR.join((parent_path, path_leaf))
lc[tc.parent_id] = tc.pk
else:
tree_path = path_leaf
ThreadedComment.objects.filter(pk=tc.pk).update(
tree_path=tree_path)
map[tc.id] = tree_path
sys.stdout.write('.')
print 'OK'
for par, child in lc.iteritems():
ThreadedComment.objects.filter(pk=par).update(last_child=child)
sys.stdout.write(':')
print
transaction.commit()
transaction.leave_transaction_management()
def handle(self, dry_run=False, **options):
verbosity = int(options['verbosity'])
verbose = verbosity > 1
# start transaction
transaction.enter_transaction_management()
transaction.managed(True)
count = 0
for document in models.Document.objects.filter(text_extracted=False):
if verbose:
print "PROCESS", repr(document)
process_document_text(document)
count += 1
if verbose:
print "\nIN SUMMARY".ljust(70, '=')
print count, "documents records processed"
print "\n"
if dry_run:
transaction.rollback()
else:
transaction.commit()
transaction.leave_transaction_management()
def _fetch_comments(self, mysql_cnx, site, object_type_ids):
"""Populates cyclope custom comments from WP table wp_comments.
instead of querying the related object for each comment and atomizing transactions, which could be expensive,
we use an additional query for each content type only, and the transaction is repeated just as many times.
we receive Site ID which is already above in the script."""
fields = ('comment_ID', 'comment_author', 'comment_author_email',
'comment_author_url', 'comment_content', 'comment_date',
'comment_author_IP', 'comment_approved', 'comment_parent',
'user_id', 'comment_post_ID')
counter = 0
for content_type_id, post_ids in object_type_ids.iteritems():
if len(post_ids) == 0: continue
query = re.sub(
"[()']", '', "SELECT {} FROM ".format(fields)
) + self.wp_prefix + "comments WHERE comment_approved!='spam' AND comment_post_ID IN {}".format(
post_ids)
cursor = mysql_cnx.cursor()
cursor.execute(query)
#single transaction per content_type
transaction.enter_transaction_management()
transaction.managed(True)
for wp_comment in cursor:
comment_hash = dict(zip(fields, wp_comment))
comment = self._wp_comment_to_custom(comment_hash, site,
content_type_id)
comment.save()
transaction.commit()
transaction.leave_transaction_management()
if cursor.rowcount > 0: counter += cursor.rowcount
cursor.close()
return counter
def save(self):
current_site = Site.objects.get_current()
transaction.enter_transaction_management()
transaction.managed(True)
try:
try:
self.user.set_password(self.cleaned_data['new_password1'])
self.user.save()
message = loader.render_to_string('reset_password_email.txt', {
'user': self.user,
'raw_password': self.cleaned_data['new_password1'],
'full_server_url': 'http://%s/' % current_site.domain
})
msg = EmailMessage(
subject='New password for CHIRP Volunteer Tracker',
body=message,
to=[self.user.email]
)
msg.send(fail_silently=False)
except:
transaction.rollback()
raise
else:
transaction.commit()
finally:
transaction.leave_transaction_management()
return self.user
def save(self):
current_site = Site.objects.get_current()
transaction.enter_transaction_management()
transaction.managed(True)
try:
try:
self.user.set_password(self.cleaned_data['new_password1'])
self.user.save()
message = loader.render_to_string(
'reset_password_email.txt', {
'user': self.user,
'raw_password': self.cleaned_data['new_password1'],
'full_server_url': 'http://%s/' % current_site.domain
})
msg = EmailMessage(
subject='New password for CHIRP Volunteer Tracker',
body=message,
to=[self.user.email])
msg.send(fail_silently=False)
except:
transaction.rollback()
raise
else:
transaction.commit()
finally:
transaction.leave_transaction_management()
return self.user
def _pre_setup(self):
if not connections_support_transactions():
fixtures = [
"sample_users", "sample_site", "sample_languages",
"sample_data"
]
if getattr(self, 'multi_db', False):
databases = connections
else:
databases = [DEFAULT_DB_ALIAS]
for db in databases:
management.call_command('flush',
verbosity=0,
interactive=False,
database=db)
management.call_command('loaddata', *fixtures, **{
'verbosity': 0,
'database': db
})
else:
if getattr(self, 'multi_db', False):
databases = connections
else:
databases = [DEFAULT_DB_ALIAS]
for db in databases:
transaction.enter_transaction_management(using=db)
transaction.managed(True, using=db)
disable_transaction_methods()
mail.outbox = []
def handle(self, *args, **options):
print "Download zip-archive..."
f = urlopen(IPGEOBASE_SOURCE_URL)
buffer = StringIO(f.read())
f.close()
print "Unpacking..."
zip_file = ZipFile(buffer)
cities_file_read = _read_file(zip_file, 'cities.txt')
cidr_optim_file_read = _read_file(zip_file, 'cidr_optim.txt')
zip_file.close()
buffer.close()
print "Start updating..."
list_cities = cities_file_read.decode(IPGEOBASE_CODING).split('\n')
list_cidr_optim = \
cidr_optim_file_read.decode(IPGEOBASE_CODING).split('\n')
lines = \
_get_cidr_optim_with_cities_lines(list_cidr_optim, list_cities)
cursor = connection.cursor()
transaction.enter_transaction_management()
try:
transaction.managed(True)
print "Delete old rows in table ipgeobase..."
cursor.execute(DELETE_SQL)
print "Write new data..."
cursor.executemany(INSERT_SQL, [l for l in lines if l])
transaction.commit()
except Exception, e:
message = "The data not updated:", e
if send_message:
mail_admins(subject=ERROR_SUBJECT, message=message)
raise CommandError, message
def test_transaction_management(self):
transaction.enter_transaction_management()
transaction.managed(True)
self.assertEqual(connection.isolation_level, self._read_committed)
transaction.leave_transaction_management()
self.assertEqual(connection.isolation_level, self._autocommit)
def install_fingerprints_and_indexes(self, target_database):
from django.db import connections
from django.db import transaction
transaction.commit_unless_managed(using=target_database)
transaction.enter_transaction_management(using=target_database)
transaction.managed(True, using=target_database)
target_conn = connections[target_database]
self.cursor = target_conn.cursor()
self.sql_context = {
'pk': UcCtab._meta.pk.name,
'mols_table': UcMols._meta.db_table,
'ctab_table': UcCtab._meta.db_table,
'fingerprints_table': UcFingerprints._meta.db_table,
'molfile': UcCtab._meta.fields[1].name,
'ctab': UcMols._meta.fields[1].name,
}
self.activate_extension()
self.create_binary_molfiles()
self.add_primary_key(self.sql_context['pk'],
self.sql_context['mols_table'])
self.install_molecular_index()
self.compute_fingerprints()
self.install_fingerprints_indexes()
self.add_primary_key(self.sql_context['pk'],
self.sql_context['fingerprints_table'])
transaction.commit(using=target_database)
transaction.leave_transaction_management(using=target_database)
def test_savepoint_localstore_flush(self):
"""
This is a very simple test to see if savepoints will actually
be committed, i.e. flushed out from localstore into cache.
"""
from django.db import transaction
transaction.enter_transaction_management()
transaction.managed()
TABLE_NAME = 'test_table'
cache_backend = johnny.cache.get_backend()
cache_backend.patch()
keyhandler = cache_backend.keyhandler
keygen = keyhandler.keygen
tm = cache_backend.cache_backend
# First, we set one key-val pair generated for our non-existing table.
table_key = keygen.gen_table_key(TABLE_NAME)
tm.set(table_key, 'val1')
# Then we create a savepoint.
# The key-value pair is moved into 'trans_sids' item of localstore.
tm._create_savepoint('savepoint1')
# We then commit all the savepoints (i.e. only one in this case)
# The items stored in 'trans_sids' should be moved back to the
# top-level dictionary of our localstore
tm._commit_all_savepoints()
# And this checks if it actually happened.
self.failUnless(table_key in tm.local)
def __call__(self, request, *args, **kwargs):
"""
By implementing the call method, we allow the class itself to act as
the view function, taking in the request object and returning the
response. This method is responsible for dealing with transaction
management, dispatching to the proper instance methods, and returning
a valid HttpResponse.
"""
if self.use_transactions:
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
context = self.view(request, *args, **kwargs)
if isinstance(context, HttpResponseRedirect):
return context
context.update(self.extra_context)
if self.next_field in request.REQUEST:
next = request.REQUEST[self.next_field]
if ':/' not in next:
return HttpResponseRedirect(next)
response = self.create_response(request, context)
if self.use_transactions:
transaction.commit()
transaction.leave_transaction_management()
return response
def handle(self, *args, **options):
self.batch_size = options.get('batch_size', 50)
self.preserve = options.get("preserve", False)
self.index = options.get("index", False)
self.newline = options.get("newline", False)
if not self.index:
old_realtime_indexing = getattr(settings, "REALTIME_INDEXING", None)
#this is not recommended by the django manual, but in case of management command it seems to work
settings.REALTIME_INDEXING = False
transaction.enter_transaction_management()
transaction.managed(True)
for records_url in args:
print("Processing %s" % records_url)
errors = self.process_url(records_url, options)
print("Processing %s Done" % records_url)
if errors:
print("%d error(s) when processing %s, check your log file." % (len(errors), records_url))
transaction.leave_transaction_management()
if not self.index and old_realtime_indexing:
settings.REALTIME_INDEXING = old_realtime_indexing
def setUp(self):
# Create a second connection to the database
self.conn2 = backend.DatabaseWrapper(**settings.DATABASE_OPTIONS)
# Put both DB connections into managed transaction mode
transaction.enter_transaction_management()
transaction.managed(True)
def test_savepoint_rollback(self):
"""Tests rollbacks of savepoints"""
from django.db import transaction
from testapp.models import Genre, Publisher
from johnny import cache
if not connection.features.uses_savepoints:
return
self.failUnless(transaction.is_managed() == False)
self.failUnless(transaction.is_dirty() == False)
connection.queries = []
cache.local.clear()
transaction.enter_transaction_management()
transaction.managed()
g = Genre.objects.get(pk=1)
start_title = g.title
g.title = "Adventures in Savepoint World"
g.save()
g = Genre.objects.get(pk=1)
self.failUnless(g.title == "Adventures in Savepoint World")
sid = transaction.savepoint()
g.title = "In the Void"
g.save()
g = Genre.objects.get(pk=1)
self.failUnless(g.title == "In the Void")
transaction.savepoint_rollback(sid)
g = Genre.objects.get(pk=1)
self.failUnless(g.title == "Adventures in Savepoint World")
transaction.rollback()
g = Genre.objects.get(pk=1)
self.failUnless(g.title == start_title)
transaction.managed(False)
transaction.leave_transaction_management()
def handle(self, *args, **options):
if not args or (args and args[0] not in ('load')):
raise CommandError("USAGE: ./manage.py %s load" % \
os.path.basename(__file__).split('.')[0])
transaction.enter_transaction_management()
transaction.managed(True)
divisions = [
'Premier League', 'Championship', 'League One', 'League Two',
'Conference'
]
tables = {}
tables['Premier League'] = scrape.parsePremierTeams()
tables['Championship'] = scrape.parseChampionship()
tables['League One'] = scrape.parseLeagueOne()
tables['League Two'] = scrape.parseLeagueTwo()
tables['Conference'] = scrape.parseConference()
for division in divisions:
for team in tables[division]:
team, division = getOrMakeTeam(team, division)
print "done", team, division
transaction.commit()
def handle(self, **options):
'''The main entry point for the Django management command.'''
import_start = datetime.datetime.now()
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
try:
self._promote_devel()
except:
self._rollback_db()
raise
# Finalize the transaction and close the db connection.
transaction.commit()
transaction.leave_transaction_management()
connection.close()
import_end = datetime.datetime.now()
# Print a short summary of what we did.
td = import_end - import_start
print '\nProcessing complete in %s days, %s.%s seconds.' % (
td.days, td.seconds, td.microseconds)
print ' TraitData objects promoted: %s' % (
PublicTraitData.objects.all().count(),)
def execute_transaction(sql, output=False, database='default'):
"A transaction wrapper for executing a list of SQL statements"
my_connection = connection
using_args = {}
if is_multi_db():
if not database:
database = DEFAULT_DB_ALIAS
my_connection = connections[database]
using_args['using'] = database
try:
# Begin Transaction
transaction.enter_transaction_management(**using_args)
transaction.managed(True, **using_args)
cursor = my_connection.cursor()
# Perform the SQL
if output:
write_sql(sql, database)
execute_sql(cursor, sql)
transaction.commit(**using_args)
transaction.leave_transaction_management(**using_args)
except Exception:
transaction.rollback(**using_args)
raise
def _fetch_attachments(self, mysql_cnx, object_type_ids):
"""Queries to WP posts table selecting attachments, not..."""
fields = ('ID', 'post_mime_type', 'guid', 'post_title', 'post_status',
'post_author', 'post_date', 'post_modified',
'comment_status', 'post_content', 'post_excerpt',
'post_parent')
query = re.sub("[()']", '', "SELECT {} FROM ".format(
fields)) + self.wp_prefix + "posts WHERE post_type='attachment'"
cursor = mysql_cnx.cursor()
cursor.execute(query)
#single transaction for all articles
transaction.enter_transaction_management()
transaction.managed(True)
for wp_post in cursor:
post = dict(zip(fields, wp_post))
attachment = self._post_to_attachment(post)
attachment.save() #whatever its type
if post['post_parent'] != 0:
relate_self, relate_other = self._relate_contents(
attachment, post['post_parent'], object_type_ids)
relate_self.save() #related contents
relate_other.save()
transaction.commit()
transaction.leave_transaction_management()
counts = (cursor.rowcount, Picture.objects.count(),
Document.objects.count(), RegularFile.objects.count(),
SoundTrack.objects.count(), MovieClip.objects.count(),
FlashMovie.objects.count(), RelatedContent.objects.count())
cursor.close()
return counts
def transaction_enter(using):
"""
トランザクション処理を開始する。それまでのcommit漏れは直前でcommitしておく。
"""
transaction.commit_unless_managed(using=using)
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
def export_sales_order(self):
transaction.enter_transaction_management(using=self.database)
transaction.managed(True, using=self.database)
try:
starttime = time()
if self.verbosity > 0:
print("Exporting requested date of sales orders...")
self.cursor.execute('''select substring(name from '^.*? '), max(plandate)
from demand
left outer join out_demand
on demand.name = out_demand.demand
and demand.subcategory = 'OpenERP'
group by substring(name from '^.*? ')
''')
cnt = 0
for i, j in self.cursor.fetchall():
result = self.sock.execute(self.openerp_db, self.uid, self.openerp_password, 'sale.order', 'write',
[int(i)], {'requested_date': j and j.strftime('%Y-%m-%d') or 0,})
cnt += 1
if self.verbosity > 0:
print("Updated %d sales orders in %.2f seconds" % (cnt, (time() - starttime)))
except Exception as e:
print("Error updating sales orders: %s" % e)
finally:
transaction.rollback(using=self.database)
transaction.leave_transaction_management(using=self.database)
def import_wetmills(task): #pragma: no cover
from .models import import_csv_wetmills
from django.db import transaction
transaction.enter_transaction_management()
transaction.managed()
try:
task.task_id = import_wetmills.request.id
task.import_log = "Started import at %s\n" % datetime.now()
task.save()
transaction.commit()
wetmills = import_csv_wetmills(task.country, task.csv_file.file.name,
task.created_by)
task.log("Import finished at %s\n" % datetime.now())
task.log("%d wetmill(s) added." % len(wetmills))
transaction.commit()
except Exception as e:
import traceback
traceback.print_exc()
task.log("Error: %s" % e)
transaction.commit()
raise e
finally:
transaction.leave_transaction_management()
return task
def atomic(using=None):
"""Perform database operations atomically within a transaction.
The caller can use this to ensure SQL statements are executed within
a transaction and then cleaned up nicely if there's an error.
This provides compatibility with all supported versions of Django.
Args:
using (str, optional):
The database connection name to use. Defaults to the default
database connection.
"""
if hasattr(transaction, 'atomic'):
# Django >= 1.5
with transaction.atomic(using=using):
yield
else:
# Django < 1.5
assert hasattr(transaction, 'enter_transaction_management')
try:
# Begin Transaction
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
yield
transaction.commit(using=using)
transaction.leave_transaction_management(using=using)
except Exception:
transaction.rollback(using=using)
raise
def serializerTest(format, self):
# Clear the database first
management.flush(verbosity=0, interactive=False)
# Create all the objects defined in the test data
objects = []
transaction.enter_transaction_management()
transaction.managed(True)
for (func, pk, klass, datum) in test_data:
objects.append(func[0](pk, klass, datum))
transaction.commit()
transaction.leave_transaction_management()
# Add the generic tagged objects to the object list
objects.extend(Tag.objects.all())
# Serialize the test database
serialized_data = serializers.serialize(format, objects, indent=2)
# Flush the database and recreate from the serialized data
management.flush(verbosity=0, interactive=False)
transaction.enter_transaction_management()
transaction.managed(True)
for obj in serializers.deserialize(format, serialized_data):
obj.save()
transaction.commit()
transaction.leave_transaction_management()
# Assert that the deserialized data is the same
# as the original source
for (func, pk, klass, datum) in test_data:
func[1](self, pk, klass, datum)
def _fixture_setup(self):
if not connections_support_transactions():
return super(TestCase, self)._fixture_setup()
assert not self.reset_sequences, 'reset_sequences cannot be used on TestCase instances'
# If the test case has a multi_db=True flag, setup all databases.
# Otherwise, just use default.
db_names = connections if getattr(self, 'multi_db', False) else [DEFAULT_DB_ALIAS]
for db_name in db_names:
transaction.enter_transaction_management(using=db_name)
transaction.managed(True, using=db_name)
disable_transaction_methods()
from django.contrib.sites.models import Site
Site.objects.clear_cache()
for db in db_names:
if hasattr(self, 'fixtures'):
call_command('loaddata', *self.fixtures,
**{
'verbosity': 0,
'commit': False,
'database': db,
'skip_validation': True,
})
def handle(self, *args, **options):
from django.db import transaction
path, slug, name = args
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
book = Book.objects.get(slug=slug)
root, ext = os.path.splitext(path)
ext = ext.lower()
if ext:
ext = ext[1:]
if ext == 'zip':
ext = 'daisy'
source_sha1 = BookMedia.read_source_sha1(path, ext)
print "Source file SHA1:", source_sha1
try:
assert source_sha1
bm = book.media.get(type=ext, source_sha1=source_sha1)
print "Replacing media: %s (%s)" % (bm.name.encode('utf-8'), ext)
except (AssertionError, BookMedia.DoesNotExist):
bm = BookMedia(book=book, type=ext)
print "Creating new media"
bm.name = name
bm.file.save(None, ExistingFile(path))
bm.save()
transaction.commit()
transaction.leave_transaction_management()
def test_transaction_management(self):
transaction.enter_transaction_management()
transaction.managed(True)
self.assertEqual(connection.isolation_level, self._read_committed)
transaction.leave_transaction_management()
self.assertEqual(connection.isolation_level, self._autocommit)
def __enter__(self):
if transaction.is_managed(self.using):
# We're already in a transaction; create a savepoint.
self.sid = transaction.savepoint(self.using)
else:
transaction.enter_transaction_management(using=self.using)
transaction.managed(True, using=self.using)
def restore(self):
self.state = BACKUP_RESTORE_STATE_IN_PROGRESS
self.save()
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
try:
self.tar = tarfile.open(self.backup_file.path, 'r:gz')
# extract data.xml for parsing
self.stream = self.tar.extractfile('backup/data.xml')
self.restore_objects()
self.restore_files()
transaction.commit()
transaction.leave_transaction_management()
self.state = BACKUP_RESTORE_STATE_DONE
except Exception as e:
transaction.rollback()
transaction.leave_transaction_management()
self.state = BACKUP_RESTORE_STATE_ERROR
self.error_message = e.__unicode__()
mail_subject = _('Restore failed')
mail_message = _('Restore for %(user)s failed with message : %(message)s') % {'user': self.user,
'message': e}
mail_admins(mail_subject, mail_message, fail_silently=(not settings.DEBUG))
self.save()
# close and delete archive
self.tar.close()
os.remove(self.backup_file.path)
self.backup_file = None
def call_test_suite_result(self, test_suite_result, name, args):
if test_suite_result not in self.test_suite_result_map:
logging.warning('Attempted to call test suite result, but not running: %s.%s', test_suite_result.id, name)
return
logging.debug('Calling test suite result: %s.%s', test_suite_result.id, name)
transaction.enter_transaction_management(True)
transaction.managed(True)
try:
getattr(self.test_suite_result_map[test_suite_result], name)(*args)
except:
transaction.rollback()
logging.exception('Test suite result failed: %s', test_suite_result.id)
self.finished_test_suite_result(test_suite_result)
try:
test_suite_result.status = 'INT'
test_suite_result.report = 'Internal error'
test_suite_result.save(force_update=True)
except:
transaction.rollback()
logging.exception('Test suite result cleanup failed: %s', test_suite_result.id)
transaction.commit()
transaction.managed(False)
transaction.leave_transaction_management()
def serializerTest(format, self):
# Clear the database first
management.call_command('flush', verbosity=0, interactive=False)
# Create all the objects defined in the test data
objects = []
transaction.enter_transaction_management()
transaction.managed(True)
for (func, pk, klass, datum) in test_data:
objects.append(func[0](pk, klass, datum))
transaction.commit()
transaction.leave_transaction_management()
# Add the generic tagged objects to the object list
objects.extend(Tag.objects.all())
# Serialize the test database
serialized_data = serializers.serialize(format, objects, indent=2)
# Flush the database and recreate from the serialized data
management.call_command('flush', verbosity=0, interactive=False)
transaction.enter_transaction_management()
transaction.managed(True)
for obj in serializers.deserialize(format, serialized_data):
obj.save()
transaction.commit()
transaction.leave_transaction_management()
# Assert that the deserialized data is the same
# as the original source
for (func, pk, klass, datum) in test_data:
func[1](self, pk, klass, datum)
def call_ranking(self, ranking, name, args):
if ranking not in self.ranking_map:
logging.warning('Attempted to call ranking, but not running: %s.%s', ranking.id, name)
return
logging.debug('Calling ranking: %s.%s', ranking.id, name)
perf.begin('ranking')
transaction.enter_transaction_management(True)
transaction.managed(True)
try:
getattr(self.ranking_map[ranking], name)(*args)
except:
transaction.rollback()
logging.exception('Ranking failed: %s', ranking.id)
self.stop_ranking(ranking)
try:
ranking.header = 'Internal error'
ranking.footer = ''
ranking.save(force_update=True)
RankingEntry.objects.filter(ranking=ranking).delete()
except:
transaction.rollback()
logging.exception('Ranking cleanup failed: %s', ranking.id)
transaction.commit()
transaction.managed(False)
transaction.leave_transaction_management()
perf.end('ranking')
def handle(self, *args, **options):
from django.db import transaction
path, slug, name = args
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
book = Book.objects.get(slug=slug)
root, ext = os.path.splitext(path)
ext = ext.lower()
if ext:
ext = ext[1:]
if ext == "zip":
ext = "daisy"
source_sha1 = BookMedia.read_source_sha1(path, ext)
print "Source file SHA1:", source_sha1
try:
assert source_sha1
bm = book.media.get(type=ext, source_sha1=source_sha1)
print "Replacing media: %s (%s)" % (bm.name.encode("utf-8"), ext)
except (AssertionError, BookMedia.DoesNotExist):
bm = BookMedia(book=book, type=ext)
print "Creating new media"
bm.name = name
bm.file.save(None, ExistingFile(path))
bm.save()
transaction.commit()
transaction.leave_transaction_management()
def _fixture_setup(self):
if not connections_support_transactions():
return super(TestCase, self)._fixture_setup()
# If the test case has a multi_db=True flag, setup all databases.
# Otherwise, just use default.
if getattr(self, 'multi_db', False):
databases = connections
else:
databases = [DEFAULT_DB_ALIAS]
for db in databases:
transaction.enter_transaction_management(using=db)
transaction.managed(True, using=db)
disable_transaction_methods()
from django.contrib.sites.models import Site
Site.objects.clear_cache()
for db in databases:
if hasattr(self, 'fixtures'):
call_command(
'loaddata', *self.fixtures, **{
'verbosity': 0,
'commit': False,
'database': db
})
def atomic(using=None):
"""Perform database operations atomically within a transaction.
The caller can use this to ensure SQL statements are executed within
a transaction and then cleaned up nicely if there's an error.
This provides compatibility with all supported versions of Django.
Args:
using (str, optional):
The database connection name to use. Defaults to the default
database connection.
"""
if hasattr(transaction, 'atomic'):
# Django >= 1.5
with transaction.atomic(using=using):
yield
else:
# Django < 1.5
assert hasattr(transaction, 'enter_transaction_management')
try:
# Begin Transaction
transaction.enter_transaction_management(using=using)
transaction.managed(True, using=using)
yield
transaction.commit(using=using)
transaction.leave_transaction_management(using=using)
except Exception:
transaction.rollback(using=using)
raise
def test_transaction_rollback(self):
"""Tests johnny's handling of transaction rollbacks.
Similar to the commit, this sets up a write to a db in a transaction,
reads from it (to force a cache write of sometime), then rolls back."""
from Queue import Queue as queue
from django.db import transaction
from testapp.models import Genre, Publisher
from johnny import cache
if settings.DATABASE_ENGINE == 'sqlite3':
print "\n Skipping test requiring multiple threads."
return
self.failUnless(transaction.is_managed() == False)
self.failUnless(transaction.is_dirty() == False)
connection.queries = []
cache.local.clear()
q = queue()
other = lambda x: self._run_threaded(x, q)
# load some data
start = Genre.objects.get(id=1)
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
# these should be the same and should have hit cache
self.failUnless(hit)
self.failUnless(ostart == start)
# enter manual transaction management
transaction.enter_transaction_management()
transaction.managed()
start.title = 'Jackie Chan Novels'
# local invalidation, this key should hit the localstore!
nowlen = len(cache.local)
start.save()
self.failUnless(nowlen != len(cache.local))
# perform a read OUTSIDE this transaction... it should still see the
# old gen key, and should still find the "old" data
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.failUnless(hit)
self.failUnless(ostart.title != start.title)
# perform a READ inside the transaction; this should hit the localstore
# but not the outside!
nowlen = len(cache.local)
start2 = Genre.objects.get(id=1)
self.failUnless(start2.title == start.title)
self.failUnless(len(cache.local) > nowlen)
transaction.rollback()
# we rollback, and flush all johnny keys related to this transaction
# subsequent gets should STILL hit the cache in the other thread
# and indeed, in this thread.
self.failUnless(transaction.is_dirty() == False)
other('Genre.objects.get(id=1)')
hit, ostart = q.get()
self.failUnless(hit)
start = Genre.objects.get(id=1)
self.failUnless(ostart.title == start.title)
transaction.managed(False)
transaction.leave_transaction_management()
def _fixture_setup(self):
if not connections_support_transactions():
return super(TestCase, self)._fixture_setup()
assert not self.reset_sequences, 'reset_sequences cannot be used on TestCase instances'
# If the test case has a multi_db=True flag, setup all databases.
# Otherwise, just use default.
db_names = connections if getattr(self, 'multi_db', False) else [DEFAULT_DB_ALIAS]
for db_name in db_names:
transaction.enter_transaction_management(using=db_name)
transaction.managed(True, using=db_name)
disable_transaction_methods()
from django.contrib.sites.models import Site
Site.objects.clear_cache()
for db in db_names:
if hasattr(self, 'fixtures'):
call_command('loaddata', *self.fixtures,
**{
'verbosity': 0,
'commit': False,
'database': db,
'skip_validation': True,
})
def handle(self, *files, **options):
connection = connections[DEFAULT_DB_ALIAS]
# Start transaction management. All fixtures are installed in a
# single transaction to ensure that all references are resolved.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
# for every connection
for connection in Connection.objects.all():
# try normalizing the number
normalized = HttpRouter.normalize_number(connection.identity)
# if it is different, then we changed it, first check to see
# if there is a connection that is identical. In that case we don't
# change anything, it is too difficult to know who might have a reference
# to this connection in the system to remap.
if normalized != connection.identity:
collision = Connection.objects.filter(identity=normalized, backend=connection.backend)
# no collision! Ok, we can just save our new identity
if not collision:
print "remapping %s to %s" % (connection.identity, normalized)
connection.identity = normalized
connection.save()
else:
print "skipping %s, collision" % (connection.identity)
transaction.commit()
def tearDown(self):
from django.db import transaction
if transaction.is_managed():
if transaction.is_dirty():
transaction.rollback()
transaction.managed(False)
transaction.leave_transaction_management()
def wrapped_func(*args, **kwargs):
enter_transaction_management(using=using)
managed(True, using=using)
try:
res = func(*args, **kwargs)
except:
if is_dirty(using=using):
rollback(using=using)
raise
else:
if is_dirty(using=using):
if not isinstance(res, HttpResponse) or res.status_code < 200 or res.status_code >= 400:
rollback(using=using)
else:
try:
commit(using=using)
except:
rollback(using=using)
raise
finally:
leave_transaction_management(using=using)
return res
def test_savepoint_localstore_flush(self):
"""
This is a very simple test to see if savepoints will actually
be committed, i.e. flushed out from localstore into cache.
"""
from django.db import transaction
transaction.enter_transaction_management()
transaction.managed()
TABLE_NAME = 'test_table'
cache_backend = johnny.cache.get_backend()
cache_backend.patch()
keyhandler = cache_backend.keyhandler
keygen = keyhandler.keygen
tm = cache_backend.cache_backend
# First, we set one key-val pair generated for our non-existing table.
table_key = keygen.gen_table_key(TABLE_NAME)
tm.set(table_key, 'val1')
# Then we create a savepoint.
# The key-value pair is moved into 'trans_sids' item of localstore.
tm._create_savepoint('savepoint1')
# We then commit all the savepoints, which should write the changes.
tm.commit()
# And this checks if it actually happened.
backend = johnny_settings._get_backend()
self.failUnless(backend.get(table_key))
def handle_noargs(self, **options):
transaction.enter_transaction_management()
transaction.managed()
for thes in Thesaurus.objects.all():
context = graph.get_context(URIRef(thes.uri))
with Term.objects.disable_mptt_updates(): # @UndefinedVariable
for i,(s,_,o) in enumerate(graph.triples((None, URIRef("http://www.w3.org/2004/02/skos/core#narrower"), None), context=context)):
print("%d - Thesaurus %s term pref label %s parent %s" % (i+1,thes.label, repr(o), repr(s)))
parent_term = Term.objects.get(uri=unicode(s)) # @UndefinedVariable
term = Term.objects.get(uri=unicode(o)) # @UndefinedVariable
term.tree_id = thes.id
term.parent = parent_term
term.save()
Term.objects.filter(parent=None, thesaurus=thes).update(tree_id=thes.id) # @UndefinedVariable
print("Rebuilding tree %d" % thes.id)
Term.objects.rebuild() # @UndefinedVariable
transaction.commit()
reset_queries()
transaction.leave_transaction_management()
def _fixture_setup(self):
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
super(SeleniumTestCase, self)._fixture_setup()
transaction.commit()
transaction.leave_transaction_management()
def pytest_runtest_setup(self, item):
item_obj = self._get_item_obj(item)
# Set the URLs if the py.test.urls() decorator has been applied
if hasattr(item.obj, "urls"):
self._old_urlconf = settings.ROOT_URLCONF
settings.ROOT_URLCONF = item.obj.urls
clear_url_caches()
# This is a Django unittest TestCase
if self._is_unittest(item_obj):
# We have to run these here since py.test's unittest plugin skips
# __call__()
item_obj.client = Client()
item_obj._pre_setup()
item_obj.setUp()
return
if not settings.DATABASE_SUPPORTS_TRANSACTIONS:
call_command("flush", verbosity=self.verbosity, interactive=not self.noinput)
else:
transaction.enter_transaction_management()
transaction.managed(True)
disable_transaction_methods()
from django.contrib.sites.models import Site
Site.objects.clear_cache()
mail.outbox = []
def __enter__(self):
if transaction.is_managed(self.using):
# We're already in a transaction; create a savepoint.
self.sid = transaction.savepoint(self.using)
else:
transaction.enter_transaction_management(using=self.using)
transaction.managed(True, using=self.using)
def handle(self, dry_run=False, **options):
verbosity = int(options['verbosity'])
verbose = verbosity > 1
# start transaction
transaction.enter_transaction_management()
transaction.managed(True)
count = 0
for document in models.Document.objects.filter(text_extracted=False):
if verbose:
print "PROCESS", repr(document)
process_document_text(document)
count += 1
if verbose:
print "\nIN SUMMARY".ljust(70, '=')
print count, "documents records processed"
print "\n"
if dry_run:
transaction.rollback()
else:
transaction.commit()
transaction.leave_transaction_management()
def _fixture_setup(self):
if not connections_support_transactions():
return super(TestCase, self)._fixture_setup()
# If the test case has a multi_db=True flag, setup all databases.
# Otherwise, just use default.
if getattr(self, 'multi_db', False):
databases = connections
else:
databases = [DEFAULT_DB_ALIAS]
for db in databases:
transaction.enter_transaction_management(using=db)
transaction.managed(True, using=db)
disable_transaction_methods()
from django.contrib.sites.models import Site
Site.objects.clear_cache()
for db in databases:
if hasattr(self, 'fixtures'):
call_command('loaddata', *self.fixtures, **{
'verbosity': 0,
'commit': False,
'database': db
})
def _fixture_setup(self):
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
super(SeleniumTestCase, self)._fixture_setup()
transaction.commit()
transaction.leave_transaction_management()
