def import_experiment_data(description):
route_split = sio.loadmat(open("{0}/outputSmallData.mat".format(DATA_PATH)))
b = np.squeeze(np.asarray(route_split['x_true']))
sql = """
UPDATE orm_experimentroute AS er SET
value = %s
WHERE vector_index = %s AND er.description = %s AND er.true_split=TRUE;
"""
values = [(v,k,description) for k, v in enumerate(b)]
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
cursor = connection.cursor()
cursor.executemany(sql, values)
transaction.commit()
transaction.set_autocommit(ac)
b = np.squeeze(np.asarray(route_split['xLBFGS']))
sql = """
UPDATE orm_experimentroute AS er SET
value = %s
WHERE vector_index = %s AND er.description = %s AND er.true_split=FALSE;
"""
values = [(v,k,description) for k, v in enumerate(b)]
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
cursor = connection.cursor()
cursor.executemany(sql, values)
transaction.commit()
transaction.set_autocommit(ac)
def import_experiment_data(description):
route_split = sio.loadmat(open("{0}/outputSmallData.mat".format(DATA_PATH)))
b = np.squeeze(np.asarray(route_split['x_true']))
sql = """
UPDATE orm_experimentroute AS er SET
value = %s
WHERE vector_index = %s AND er.description = %s AND er.true_split=TRUE;
"""
values = [(v,k,description) for k, v in enumerate(b)]
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
cursor = connection.cursor()
cursor.executemany(sql, values)
transaction.commit()
transaction.set_autocommit(ac)
b = np.squeeze(np.asarray(route_split['xLBFGS']))
sql = """
UPDATE orm_experimentroute AS er SET
value = %s
WHERE vector_index = %s AND er.description = %s AND er.true_split=FALSE;
"""
values = [(v,k,description) for k, v in enumerate(b)]
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
cursor = connection.cursor()
cursor.executemany(sql, values)
transaction.commit()
transaction.set_autocommit(ac)
def autocommit_6(request):
autocommit_stat = transaction.get_autocommit()
print("autocommit status : %s" %
(autocommit_stat, )) # autocommit could be True or False
with transaction.atomic():
autocommit_stat2 = transaction.get_autocommit()
print("autocommit status : %s" % (
autocommit_stat2, )) # in atomic block, autocommit is always False
def inner(**kwargs):
try:
get_autocommit()
except:
yield checks.Warning(
"Database not reachable",
hint="Try running bootstrap again",
id="promgen.W000",
)
else:
yield from func(**kwargs)
def syncWithSettings():
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
try:
# Bring the scenario table in sync with settings.databases
dbs = [i for i, j in settings.DATABASES.items() if j['NAME']]
for sc in Scenario.objects.all():
if sc.name not in dbs:
sc.delete()
scs = [sc.name for sc in Scenario.objects.all()]
for db in dbs:
if db not in scs:
if db == DEFAULT_DB_ALIAS:
Scenario(name=db,
status="In use",
description='Production database').save()
else:
Scenario(name=db, status="Free").save()
transaction.commit()
except Exception as e:
logger.error(
"Error synchronizing the scenario table with the settings: %s"
% e)
transaction.rollback()
finally:
transaction.set_autocommit(ac)
def destroy_char_version_config(self,
db_write_back: Literal[False]) -> None:
"""
Needs to be run before the char version (config) is deleted from the database. This informs managers
that they should do some cleanup (typically nothing is required, as the DB will do the cleanup);
this function pairs with create_char_version_config.
Note that we do NOT call the individual managers' removal-from-config hooks.
"""
assert not db_write_back
if transaction.get_autocommit(
): # Essentially always a bug. Note that django.test.TestCase wraps everything.
config_logger.critical(
"Calling destroy_char_version_config outside a transaction")
warnings.warn(
"destroy_char_version_config should be wrapped in a transaction.",
RuntimeWarning)
if self.db_char_version is None: # This makes little sense outside of specific testing instances.
conditional_log(
config_logger,
'Called destroy_char_version_config without an associated db entry',
normal_level='critical',
test_level='info')
self._re_init(setup_managers=False
) # Note that this validates the syntax before deletion.
self.setup_managers(create=CreateManagerEnum.destroy_config)
def atomic_9(resquest):
g1 = Group()
s_info = 'atomic_9 save 1'
g1.name = '%s %s' % (app_prefix, s_info)
g1.save()
with transaction.atomic():
g2 = Group()
s_info = 'atomic_9 save 2'
g2.name = '%s %s' % (app_prefix, s_info)
g2.save()
g3 = Group()
s_info = 'atomic_9 save 3'
g3.name = '%s %s' % (app_prefix, s_info)
g3.save()
auto_commit = transaction.get_autocommit()
#transaction.set_autocommit(auto_commit) # raising an exception, it is forbidden when atomic block is active
#transaction.rollback() # raising an exception, it is forbidden when atomic block is active
print("function get_autocommit() : %s" % (auto_commit, ))
sid = transaction.savepoint() # test
print("save point id %s" % (sid, )) # test
transaction.savepoint_rollback(sid)
transaction.clean_savepoints()
rollback_cond = transaction.get_rollback()
print("rollback condition is %s" % (rollback_cond, ))
transaction.set_rollback(rollback_cond)
raise Exception("Error 2")
def is_managed(using=None):
if django.VERSION[:2] < (1, 6):
return transaction.is_managed(using=using)
elif django.VERSION[:2] >= (1, 6):
# See https://code.djangoproject.com/ticket/21004
return not transaction.get_autocommit(using=using)
return False
def handle(self, *sqlfiles, **options):
self.verbosity = options['verbosity']
db = options['database']
self.conn = connections[db]
sf_readers = {sf: self.get_reader(sf) for sf in sqlfiles}
for sqlfile, reader in sf_readers.items():
self.vprint(2, 'Processing file {}...'.format(sqlfile))
self.vprint(2, '...opening file...')
opener, mode = reader
self.vprint(3, '...using reader {}...'.format(opener))
with opener(sqlfile, mode) as s:
sql = s.read()
self.vprint(2, 'File opened.')
self.vprint(2, 'Running sql...')
if options['use_transaction']:
with transaction.atomic(using=db):
self.vprint(
2,
'...opened transaction...',
)
self.runsql(sql)
else:
self.runsql(sql)
self.vprint(2, 'sql completed.')
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(db):
self.conn.close()
def fetchOne(self, sql: str, params: dict = dict()) -> List[dict]:
with connection.cursor() as cursor:
self.execute_retornando_cursor(sql, params, cursor)
retorno = CursorUtil().fetchone(cursor)
if transaction.get_autocommit():
connection.close()
return retorno
def import_experiment_from_agent_google_match():
import uuid
from django.db import transaction
from phidb.db.backends.postgresql_psycopg2.base import *
from django.db import connection
import numpy as np
with server_side_cursors(connection):
cursor = connection.cursor()
query = """
"""
cursor.execute(query)
results = [row for row in cursor]
results = [(i,o,d,r,v) for (i,o,d,r,v) in results]
print 'len: %s' % len(results)
description = 'agents_google_match'
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for j,(i,o,d,r,v) in enumerate(results):
er = ExperimentRoute(route=i, vector_index=j, value=v,
description=description, true_split=True)
er.save()
transaction.commit()
transaction.set_autocommit(ac)
def handle(self, *jsonfiles, **options):
self.verbosity = options['verbosity']
db = options['database']
self.conn = connections[db]
jf_readers = {jf: self.get_reader(jf) for jf in jsonfiles}
for jsonfile, reader in jf_readers.items():
self.vprint(2, 'Processing file {}...'.format(jsonfile))
self.vprint(2, '...opening file...')
opener, mode = reader
self.vprint(3, '...using reader {}...'.format(opener))
with opener(jsonfile, mode) as f:
_json = json.load(f)
self.vprint(2, 'File opened.')
self.vprint(2, 'Importing json...')
if options['use_transaction']:
with transaction.atomic(using=db):
self.vprint(
2,
'...opened transaction...',
)
self.import_json(_json)
else:
self.import_json(_json)
self.vprint(2, 'import completed.')
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(db):
self.conn.close()
def setUp(self):
warnings.simplefilter('ignore')
self.building = Building.objects.get(pk=2) # 表示項目確認用マンション
self.facility = self.building.building_facilities.first()
if transaction.get_autocommit():
transaction.set_autocommit(False)
def setUp(self):
warnings.simplefilter('ignore')
self.room = Room.objects.get(pk=3) # 表示項目確認用マンション DEMO1号室
self.movie = self.room.room_movies.first()
if transaction.get_autocommit():
transaction.set_autocommit(False)
def is_managed(using=None):
if django.VERSION[:2] < (1, 6):
return transaction.is_managed(using=using)
elif django.VERSION[:2] >= (1, 6):
# See https://code.djangoproject.com/ticket/21004
return not transaction.get_autocommit(using=using)
return False
def handle(self, *args, **options):
saved_ac = get_autocommit()
try:
set_autocommit(False)
for path in args:
with open(path, 'r') as f:
for l in f:
try:
source_id, text = l.rstrip().split('\t')
tokens = text.strip().split(' ')
targets, _, _ = analyse_tokens(tokens)
if len(targets) > 0:
sent = Sentence.objects.create(
source_id=source_id, text=text)
sent.save()
else:
print(source_id, 'does not have connectives')
except IntegrityError:
print(source_id, 'cannot be inserted')
commit()
except:
raise
finally:
set_autocommit(saved_ac)
def test_atomic_prevents_setting_autocommit(self):
autocommit = transaction.get_autocommit()
with transaction.atomic():
with self.assertRaises(transaction.TransactionManagementError):
transaction.set_autocommit(not autocommit)
# Make sure autocommit wasn't changed.
self.assertEqual(connection.autocommit, autocommit)
def import_experiment_from_agent_google_match():
import uuid
from django.db import transaction
from phidb.db.backends.postgresql_psycopg2.base import *
from django.db import connection
import numpy as np
with server_side_cursors(connection):
cursor = connection.cursor()
query = """
"""
cursor.execute(query)
results = [row for row in cursor]
results = [(i,o,d,r,v) for (i,o,d,r,v) in results]
print 'len: %s' % len(results)
description = 'agents_google_match'
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for j,(i,o,d,r,v) in enumerate(results):
er = ExperimentRoute(route=i, vector_index=j, value=v,
description=description, true_split=True)
er.save()
transaction.commit()
transaction.set_autocommit(ac)
def notify(obj, **kwargs):
if not transaction.get_autocommit():
# we're inside a managed transaction.
# send the notification only after successful commit
transaction.on_commit(lambda: events.notify(obj, **kwargs))
else:
events.notify(obj, **kwargs)
def test_atomic_prevents_setting_autocommit(self):
autocommit = transaction.get_autocommit()
with transaction.atomic():
with self.assertRaises(transaction.TransactionManagementError):
transaction.set_autocommit(not autocommit)
# Make sure autocommit wasn't changed.
self.assertEqual(connection.autocommit, autocommit)
def forwards(self, orm):
from django.db import transaction
prevCommit = transaction.get_autocommit()
transaction.set_autocommit(True)
for s in orm['sites.Site'].objects.all():
orm['local.SpaceConfig'].objects.get_or_create(site=s)
transaction.set_autocommit(prevCommit)
def add_jobs(self, jobs, command):
"""Add a job, and any others which are required in order to reach its prerequisite state"""
# Important: the Job must not be committed until all
# its dependencies and locks are in.
assert not transaction.get_autocommit()
for job in jobs:
for dependency in self._dep_cache.get(job).all():
if not dependency.satisfied():
log.info("add_jobs: setting required dependency %s %s" %
(dependency.stateful_object,
dependency.preferred_state))
self._set_state(dependency.get_stateful_object(),
dependency.preferred_state, command)
log.info("add_jobs: done checking dependencies")
locks = self._create_locks(job)
job.locks_json = json.dumps([l.to_dict() for l in locks])
self._create_dependencies(job, locks)
job.save()
log.info("add_jobs: created Job %s (%s)" %
(job.pk, job.description()))
for l in locks:
self._lock_cache.add(l)
command.jobs.add(job)
self._job_collection.add_command(command, jobs)
def perform_destroy(self, instance: Zaak):
if instance.besluit_set.exists():
raise ValidationError(
{
api_settings.NON_FIELD_ERRORS_KEY:
_("All related Besluit objects should be destroyed before destroying the zaak"
)
},
code="pending-besluit-relation",
)
# check if we need to delete any remote OIOs
autocommit = transaction.get_autocommit()
assert autocommit is False, "Expected to be in a transaction.atomic block"
# evaluate the queryset, because the transaction will delete the records with
# a cascade
oio_urls = instance.zaakinformatieobject_set.filter(
_informatieobject__isnull=True).values_list(
"_objectinformatieobject_url", flat=True)
delete_params = [(url, Service.get_client(url)) for url in oio_urls]
def _delete_oios():
for url, client in delete_params:
client.delete("objectinformatieobject", url=url)
transaction.on_commit(_delete_oios)
super().perform_destroy(instance)
def nocommit(using=None):
backup = t.get_autocommit(using)
t.set_autocommit(False, using)
t.enter_transaction_management(managed=True, using=using)
yield
t.rollback(using)
t.leave_transaction_management(using)
t.set_autocommit(backup, using)
def on_commit(self, func):
if self.in_atomic_block \
or (not transaction.get_autocommit() and transaction.is_dirty()): # for backward compatibility with old transaction system
# transaction in progress; save for execution on commit
self.run_on_commit.append((self.savepoint_ids[:], func))
else:
# no transaction in progress; execute immediately
func()
def nocommit(using=None):
backup = t.get_autocommit(using)
t.set_autocommit(False, using)
t.enter_transaction_management(managed=True, using=using)
yield
t.rollback(using)
t.leave_transaction_management(using)
t.set_autocommit(backup, using)
def autocommit_3(request):
autocommit_flag = transaction.get_autocommit()
print("auto commit state : %s" % (autocommit_flag, ))
#transaction.set_autocommit(autocommit_flag)
g1 = Group()
s_info = 'autocommit 3'
g1.name = '%s %s' % (app_prefix, s_info)
g1.save()
def write_back_to_db(self, /) -> None:
if transaction.get_autocommit():
raise RuntimeError(
"writing to db only allowed inside a transaction.")
# This should really do nothing.
self.db_char_version.refresh_from_db()
self._json_recipe = None
self.db_char_version.json_config = self.json_recipe
self.db_char_version.save()
def import_lookup(verbose=True):
waypoints = pickle.load(open("{0}/lookup.pickle".format(DATA_PATH)))
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for matrix_id, taz_id in waypoints.iteritems():
mt = MatrixTaz(matrix_id=matrix_id, taz_id=taz_id)
mt.save()
transaction.commit()
transaction.set_autocommit(ac)
def get_connection(list_key=Conf.PREFIX):
if transaction.get_autocommit(): # Only True when not in an atomic block
# Make sure stale connections in the broker thread are explicitly
# closed before attempting DB access.
# logger.debug("Broker thread calling close_old_connections")
db.close_old_connections()
else:
logger.debug("Broker in an atomic transaction")
return OrmQ.objects.using(Conf.ORM)
def import_lookup(verbose=True):
waypoints = pickle.load(open("{0}/lookup.pickle".format(DATA_PATH)))
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for matrix_id, taz_id in waypoints.iteritems():
mt = MatrixTaz(matrix_id=matrix_id, taz_id=taz_id)
mt.save()
transaction.commit()
transaction.set_autocommit(ac)
def get_connection(list_key=Conf.PREFIX):
if transaction.get_autocommit(): # Only True when not in an atomic block
# Make sure stale connections in the broker thread are explicitly
# closed before attempting DB access.
# logger.debug("Broker thread calling close_old_connections")
db.close_old_connections()
else:
logger.debug("Broker in an atomic transaction")
return OrmQ.objects.using(Conf.ORM)
def handle(self, *files, **options):
objs_with_deferred_fields = []
# temporarily hard code to default
objs_with_self_deferred_fields = []
# this is for the model contain the self references
file_paths = []
for f in files:
file_paths.extend(_auto_find_chunk_files_with_abs_path(f))
with transaction.atomic():
with DisableForeignkeyConstrain(connection):
for file_path in file_paths:
logger.info(f'start processing {file_path}')
deferred_objs = _bulk_create_from_fixture(file_path)
objs_with_deferred_fields.extend(deferred_objs)
logger.info(f'establish all deferred fields')
for obj in objs_with_deferred_fields:
try:
with transaction.atomic():
obj.save_deferred_fields()
except IntegrityError as e:
# It may encounter redundant object if you load game data
# with lots of different model (ex. common foreign model)
# so I bypass this error
# NOTE: some models contains self-reference foreign key so
# we should collect them and update it later
objs_with_self_deferred_fields.append(obj)
if len(objs_with_self_deferred_fields) > 0:
with transaction.atomic():
# query the instances needed to update
for obj in objs_with_self_deferred_fields:
cls = obj.object.__class__
fields_to_filter = {}
if cls._meta.unique_together:
for f in cls._meta.unique_together:
fields_to_filter[f] = None
else:
for field in cls._meta.fields:
if field.unique and field.attname != 'id':
fields_to_filter[field.attname] = None
break
for f in fields_to_filter:
fields_to_filter[f] = getattr(obj.object, f)
obj.object = cls.objects.get(**fields_to_filter)
obj.save_deferred_fields()
if transaction.get_autocommit():
connection.close()
def read_loose_ref(self, name):
qs = Refs.objects.only('value')
if not get_autocommit(using=qs._db):
qs = qs.select_for_update()
try:
ref = qs.get(repo=self._repo, ref=name)
except Refs.DoesNotExist:
return None
else:
return ref.value
def read_loose_ref(self, name):
qs = Refs.objects.only('value')
if not get_autocommit(using=qs._db):
qs = qs.select_for_update()
try:
ref = qs.get(repo=self._repo, ref=name)
except Refs.DoesNotExist:
return None
else:
return ref.value
def create_char_version_config(cls,
*,
from_python: PythonConfigRecipe = None,
from_json: str = None,
char_version: BaseCharVersion = None,
db_char_version: CharVersionModel = None,
setup_managers: Literal[True],
db_write_back: Literal[False]) -> CVConfig:
"""
Called upon creating a new char in the database, returns a new char_version_config for this.
Note that the difference to calling CVConfig(...) is that CVConfig provides an interface to a Config / db entry
which is serialized in the database, whereas create_char_version_config is used when creating it.
It calls on-create hooks on the managers, which may touch the database. Create_char_version_config itself
does NOT touch the database.
This should be called EXACTLY ONCE for a given (db version of) char_version config and the result saved in the db.
Note that uuids and reference validity need to be correct before calling this function. The on-create hooks
are ill-suited to fix / change the config in that regard; The purpose of this function is really only preparing
db entries. If fixing references is requires, add the relevant managers.
The db_char_version entry must already exist in the db (but can be a minimal / semantically invalid entry) and
its config entry will have to be overwritten using the result of this call within the same db transaction.
This is because managers may need the database primary key to create db references upon setup.
"""
# TODO: Reconsider forcing db_write_back == False.
assert not db_write_back
assert setup_managers # for now. Create_char_version_config always runs setup, essentially just to run checks.
if transaction.get_autocommit(
): # Essentially always a bug. Note that django.test.TestCase wraps everything.
config_logger.critical(
"Calling create_char_version_config outside a transaction")
warnings.warn(
"create_char_version_config should be wrapped in a transaction.",
RuntimeWarning)
# NOTE: the cls constructor stores a (deep!)copy of from_python, if set
new_char_version_config = cls(from_python=from_python,
from_json=from_json,
char_version=char_version,
db_char_version=db_char_version,
setup_managers=False,
validate_setup=False,
validate_syntax=True)
if new_char_version_config.db_char_version is None: # This case makes little sense outside of specific testing instances.
conditional_log(
config_logger,
'Called create_char_version_config without and associated db entry',
normal_level='critical',
test_level='info')
new_char_version_config.setup_managers(
create=CreateManagerEnum.create_config)
new_char_version_config._re_init(setup_managers=setup_managers)
return new_char_version_config
def savepoint_4(request):
autocommit_stat = transaction.get_autocommit()
print("autocommit status : %s" % (autocommit_stat, ))
#rollback_stat = transaction.get_rollback() #The rollback flag doesn't work outside of an 'atomic' block
#print("rollback status : %s" % (rollback_stat,))
#transaction.set_rollback(False)
sid1 = transaction.savepoint()
g1 = Group()
s_info = 'savepoint_4 a'
g1.name = '%s %s' % (app_prefix, s_info)
g1.save()
g2 = Group()
s_info = 'savepoint_4 b'
g2.name = '%s %s' % (app_prefix, s_info)
g2.save()
transaction.savepoint_commit(sid1)
sid2 = transaction.savepoint()
g3 = Group()
s_info = 'savepoint_4 c'
g3.name = '%s %s' % (app_prefix, s_info)
g3.save()
g4 = Group()
s_info = 'savepoint_4 d'
g4.name = '%s %s' % (app_prefix, s_info)
g4.save()
transaction.savepoint_rollback(sid2) # not working
sid3 = transaction.savepoint()
g5 = Group()
s_info = 'savepoint_4 e'
g5.name = '%s %s' % (app_prefix, s_info)
g5.save()
g6 = Group()
s_info = 'savepoint_4 f'
g6.name = '%s %s' % (app_prefix, s_info)
g6.save()
transaction.savepoint_commit(sid3)
transaction.clean_savepoints()
raise Exception("Error")
def import_waypoints(verbose=True):
waypoints = pickle.load(open("{0}/Phi/waypoints/waypoints-950.pkl".format(DATA_PATH)))
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for category, locations in waypoints.iteritems():
for location in locations:
pt = Point(tuple(location), srid=canonical_projection)
wp = Waypoint(location=pt, location_dist=pt.transform(google_projection, clone=True), category=category)
wp.save()
transaction.commit()
transaction.set_autocommit(ac)
def import_cell_towers(verbose=True):
towers = sio.loadmat("data/tower_locations.mat")
towers = towers['towers']
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for location in towers:
pt = Point(tuple(location), srid=4326)
wp = Tower(location=pt, location_dist=pt.transform(900913, clone=True), category="From tower_locations.mat")
wp.save()
transaction.commit()
transaction.set_autocommit(ac)
def load_waypoints_file(filepath, density_id):
waypoints = pickle.load(open("{0}/{1}".format(config.WAYPOINTS_DIRECTORY, filepath)))
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for category, locations in waypoints.iteritems():
for location in locations:
pt = Point(tuple(location), srid=canonical_projection)
wp = Waypoint(location=pt, location_dist=pt.transform(google_projection, clone=True), category=category, density_id=density_id)
wp.save()
transaction.commit()
transaction.set_autocommit(ac)
def handle(self, *args, **options):
self.using = options['database']
with transaction.atomic(using=self.using):
reload_stock_queries()
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(self.using):
connections[self.using].close()
def setUp(self):
warnings.simplefilter('ignore')
self.client = Client()
if transaction.get_autocommit():
transaction.set_autocommit(False)
response = self.client.post(reverse('login'), {
'username': '******',
'password': '******',
},
follow=True)
self.assertEqual(response.status_code, 200)
def savepoint_2(request):
autocommit_flag = transaction.get_autocommit()
print("autocommit state : %s" % (autocommit_flag, ))
rollback_stat = transaction.get_rollback()
print("rollback status : %s" % (rollback_stat, ))
sid1 = transaction.savepoint()
g1 = Group()
s_info = 'savepoint_2 a'
g1.name = '%s %s' % (app_prefix, s_info)
g1.save()
g2 = Group()
s_info = 'savepoint_2 b'
g2.name = '%s %s' % (app_prefix, s_info)
g2.save()
transaction.savepoint_commit(sid1)
sid2 = transaction.savepoint()
g3 = Group()
s_info = 'savepoint_2 c'
g3.name = '%s %s' % (app_prefix, s_info)
g3.save()
g4 = Group()
s_info = 'savepoint_2 d'
g4.name = '%s %s' % (app_prefix, s_info)
g4.save()
transaction.savepoint_rollback(sid2)
sid3 = transaction.savepoint()
g5 = Group()
s_info = 'savepoint_2 e'
g5.name = '%s %s' % (app_prefix, s_info)
g5.save()
g6 = Group()
s_info = 'savepoint_2 f'
g6.name = '%s %s' % (app_prefix, s_info)
g6.save()
transaction.savepoint_commit(sid3)
transaction.clean_savepoints()
raise Exception("Error")
def apply_async(cls, *args, **kwargs):
# Delay the task unless the client requested otherwise or transactions
# aren't being managed (i.e. the signal handlers won't send the task).
if not transaction.get_autocommit():
if not transaction.is_dirty():
# Always mark the transaction as dirty
# because we push task in queue that must be fired or discarded
if "using" in kwargs:
transaction.set_dirty(using=kwargs["using"])
else:
transaction.set_dirty()
_get_task_queue().append((cls, args, kwargs))
else:
return cls.original_apply_async(*args, **kwargs)
def createTable(self):
"""
Sets up the database table using self.field_list
"""
if not self.field_list:
self._getModelFieldList()
if transaction.get_autocommit():
with transaction.atomic():
with connection.schema_editor() as schema_editor:
schema_editor.create_model(self.createDynModel())
else:
with connection.schema_editor() as schema_editor:
schema_editor.create_model(self.createDynModel())
def load(self, filename, dryrun=True):
directory, module_name = os.path.split(filename)
module_name = os.path.splitext(module_name)[0]
sys.path.insert(0, directory)
try:
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
with reversion.create_revision():
try:
__import__(module_name)
except Exception as e:
raise ImportError(u"Failed to Import - Exception: %s" % e)
except ImportError, e:
raise
def handle(self, *fixture_labels, **options):
self.ignore = options['ignore']
self.using = options['database']
self.app_label = options['app_label']
self.verbosity = options['verbosity']
self.excluded_models, self.excluded_apps = parse_apps_and_model_labels(options['exclude'])
with transaction.atomic(using=self.using):
self.loaddata(fixture_labels)
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(self.using):
connections[self.using].close()
def handle(self, *args, **options):
self.style = no_style()
self.using = options['database']
fixture_labels = options['fixture_labels'].split(',') if options['fixture_labels'] else ()
try:
with transaction.atomic():
self.syncdata(fixture_labels, options)
except SyncDataError as exc:
print(self.style.ERROR(exc))
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(self.using):
connections[self.using].close()
def import_experiment_sensors(description):
"""Assumes you have an experiment with that description"""
# Load b vector from MAT file
route_split = sio.loadmat(open("{0}/experiment_matrices/route_assignment_matrices_ntt.mat".format(DATA_PATH)))
b = np.squeeze(np.asarray(route_split['b']))
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
# Find experiment
experiment = Experiment.objects.get(description=description)
# Load CSV, enumerate over lines
for idx, row in enumerate(csv.DictReader(open("{0}/sensors.csv".format(DATA_PATH)))):
sensor = Sensor.objects.get(pems_id=row['ID'], road_type='Freeway')
es = ExperimentSensor(sensor=sensor, value=b[idx], experiment=experiment, vector_index=idx)
es.save()
# For each line, find value in b, and then create the ExperimentSensor with that vector_index cross-checked by PEMS id
transaction.commit()
transaction.set_autocommit(ac)
def handle(self, *fixture_labels, **options):
self.ignore = options.get('ignore')
self.using = options.get('database')
self.app_label = options.get('app_label')
self.hide_empty = options.get('hide_empty', False)
self.verbosity = options.get('verbosity')
with transaction.atomic(using=self.using):
self.loaddata(fixture_labels)
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(self.using):
connections[self.using].close()
def import_experiment(filename, description):
back_map = pickle.load(open(filename))
condensed_map = defaultdict(list)
for k, v in back_map.iteritems():
condensed_map[v].append(k)
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for k, v in condensed_map.iteritems():
o, d = k
for idx, rt in enumerate(v):
route = find_route_by_origin_destination_route_index(o, d, idx)
er = ExperimentRoute(route=route, vector_index=rt, description=description, true_split=False)
er.save()
er = ExperimentRoute(route=route, vector_index=rt, description=description, true_split=True)
er.save()
transaction.commit()
transaction.set_autocommit(ac)
def import_nodes(verbose=True):
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for idx, row in enumerate(csv.DictReader(open("{0}/Nodes_I15.csv".format(DATA_PATH)))):
try:
params = {node_mapping[k]:v for k, v in row.iteritems() if (k and v and k in node_mapping)}
params = {k: link_mapping_type[k](v) for k, v in params.iteritems() if k in link_mapping_type}
geomstr = 'SRID=4326;POINT('+row['GEOMSTR']+')'
params['geom'] = GEOSGeometry(geomstr)
params['geom_dist'] = params['geom'].transform(900913, clone=True)
node = models.Node(**params)
node.save()
except:
logging.error("Error parsing on line %d: row is \"%s\" and params extracted are \"%s\"" % (idx+1, params, row))
transaction.rollback()
transaction.set_autocommit(ac)
raise
transaction.commit()
transaction.set_autocommit(ac)
def import_links(verbose=True):
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
for idx, row in enumerate(csv.DictReader(open("{0}/Links_I15.csv".format(DATA_PATH)))):
try:
params = {k:v for k, v in row.iteritems() if (k and v)}
params = {k: link_mapping_type[k](v) for k, v in params.iteritems() if k in link_mapping_type}
geomstr = row['geom']
params['geom'] = GEOSGeometry(geomstr)
params['geom_dist'] = params['geom'].transform(900913, clone=True)
link = models.Link(**params)
link.save()
except:
print idx
print params
print row
raise
transaction.commit()
transaction.set_autocommit(ac)
def lock(flow_task, process_pk):
assert transaction.get_autocommit() or transaction.commit.__module__ == 'django.test.testcases'
with transaction.atomic():
for i in range(attempts):
try:
flow_task.flow_cls.process_cls._default_manager \
.filter(pk=process_pk) \
.select_for_update(nowait=nowait) \
.exists()
break
except DatabaseError:
if i != attempts-1:
sleep_time = (((i+1)*random.random()) + 2**i) / 2.5
time.sleep(sleep_time)
else:
raise FlowLockFailed('Lock failed for {}'.format(flow_task.name))
yield
def file_handler(xml_file):
auto_commit_status = transaction.get_autocommit()
transaction.set_autocommit(False)
value = xml_file.read()
tree = etree.fromstring(value)
operators = {}
for item in Operator.objects.all():
operators[item.code] = item
result = []
for item in tree.xpath('//td/c'):
get_value = lambda xpath: item.xpath(xpath)[0].text
data_record = DataRecord()
data_record.time = datetime.strptime(get_value('./d'), FORMAT_TIME)
data_type = get_value('./s')
data_type = data_type if data_type is not None else ''
duration = get_value('./du')
if 'INTERNET' in get_value('./n'):
data_record.data_type = INTERNET
data_record.duration = duration[:-2] # Remove 'Kb' from string
else:
data_record.operator = operators.get('U')
source = get_value('./n')
for operator_code in operators.keys():
if operator_code in source:
data_record.operator = operators.get(operator_code)
break
data_record.number = re.findall(NUMBER_PATTERN, get_value('./n'))[0]
data_record.direction = INCOMING if '<--' in get_value('./n') else OUTGOING
if 'sms' in data_type:
data_record.data_type = SMS
data_record.duration = duration
elif 'ussd' in data_type:
continue
else:
data_record.data_type = CALL
call_duration = datetime.strptime(duration, DURATION_TIME)
data_record.duration = call_duration.minute * 60 + call_duration.second
data_record.save()
result.append(data_record)
transaction.commit()
transaction.set_autocommit(auto_commit_status)
return result
def test_mark_for_rollback_on_error_in_autocommit(self):
self.assertTrue(transaction.get_autocommit())
# Swallow the intentional error raised.
with self.assertRaisesMessage(Exception, "Oops"):
# Wrap in `mark_for_rollback_on_error` to check if the transaction is marked broken.
with transaction.mark_for_rollback_on_error():
# Ensure that we are still in a good state.
self.assertFalse(transaction.get_connection().needs_rollback)
raise Exception("Oops")
# Ensure that `mark_for_rollback_on_error` did not mark the transaction
# as broken, since we are in autocommit mode …
self.assertFalse(transaction.get_connection().needs_rollback)
# … and further queries work nicely.
Reporter.objects.create()
def handle(self, *fixture_labels, **options):
self.ignore = options.get("ignore")
self.using = options.get("database")
if not len(fixture_labels):
raise CommandError(
"No database fixture specified. Please provide the path of at " "least one fixture in the command line."
)
self.verbosity = int(options.get("verbosity"))
with transaction.commit_on_success_unless_managed(using=self.using):
self.loaddata(fixture_labels)
# Close the DB connection -- unless we're still in a transaction. This
# is required as a workaround for an edge case in MySQL: if the same
# connection is used to create tables, load data, and query, the query
# can return incorrect results. See Django #7572, MySQL #37735.
if transaction.get_autocommit(self.using):
connections[self.using].close()
def syncWithSettings():
ac = transaction.get_autocommit()
transaction.set_autocommit(False)
try:
# Bring the scenario table in sync with settings.databases
dbs = [ i for i,j in settings.DATABASES.items() if j['NAME'] ]
for sc in Scenario.objects.all():
if sc.name not in dbs:
sc.delete()
scs = [sc.name for sc in Scenario.objects.all()]
for db in dbs:
if db not in scs:
if db == DEFAULT_DB_ALIAS:
Scenario(name=db, status=u"In use", description='Production database').save()
else:
Scenario(name=db, status=u"Free").save()
transaction.commit()
except Exception as e:
logger.error("Error synchronizing the scenario table with the settings: %s" % e)
transaction.rollback()
finally:
transaction.set_autocommit(ac)
def process_response(self, request, response):
"""Commits and leaves transaction management."""
if not transaction.get_autocommit():
if transaction.is_dirty():
# Note: it is possible that the commit fails. If the reason is
# closed connection or some similar reason, then there is
# little hope to proceed nicely. However, in some cases (
# deferred foreign key checks for exampl) it is still possible
# to rollback().
try:
transaction.commit()
except Exception:
# If the rollback fails, the transaction state will be
# messed up. It doesn't matter, the connection will be set
# to clean state after the request finishes. And, we can't
# clean the state here properly even if we wanted to, the
# connection is in transaction but we can't rollback...
transaction.rollback()
transaction.leave_transaction_management()
raise
transaction.leave_transaction_management()
return response
def handle(self, *args, **options):
saved_ac = get_autocommit()
try:
set_autocommit(False)
for path in args:
with open(path, 'r') as f:
for l in f:
try:
tokens = l.strip().split('\t')
if len(tokens) == 1:
single = SingleConnective.objects.create(text=tokens[0])
single.save()
elif len(tokens) == 2:
pair = PairConnective.objects.create(first_half=tokens[0], second_half=tokens[1])
pair.save()
except IntegrityError:
print(l, 'cannot be inserted')
commit()
except:
raise
finally:
set_autocommit(saved_ac)
