def FetchOrCreateItem(summary): """Find an exisiting entry of this result, or create a new entry""" key = ResultCommonEntryProtocolSet._CalculateKey(summary) if key in ResultCommonEntryProtocolSet.__cache: return ResultCommonEntryProtocolSet.__cache[key]; while True: try: sid = transaction.savepoint() item,created= ResultCommonEntryProtocolSet.objects.get_or_create(key=key) if created: item.protcol_result_summary.add(*summary) transaction.savepoint_commit(sid) except DatabaseError: transaction.savepoint_rollback(sid) time.sleep(0.1) continue except IntegrityError: transaction.savepoint_rollback(sid) time.sleep(0.1) continue except: transaction.savepoint_rollback(sid) transaction.rollback_unless_managed() raise break; ResultCommonEntryProtocolSet.__cache[key] = item return item;
def FetchOrCreateItem(**params):
"""Find an existing result entry, or create a new entry"""
param1 = dict([(x,params[x]) for x in ["version_tested_major","version_tested_minor","result_summary_group"]])
key = "{version_tested_major:d}{version_tested_minor:d}_{result_summary_group.id:x}".format(**param1)
if key in ResultCommonEntryProtocol.__cache:
return ResultCommonEntryProtocol.__cache[key];
while True:
try:
sid = transaction.savepoint()
item,created= ResultCommonEntryProtocol.objects.get_or_create(**param1)
transaction.savepoint_commit(sid)
except DatabaseError:
transaction.savepoint_rollback(sid)
time.sleep(0.1)
continue
except IntegrityError:
transaction.savepoint_rollback(sid)
time.sleep(0.1)
continue
except:
transaction.savepoint_rollback(sid)
transaction.rollback_unless_managed()
raise
break;
ResultCommonEntryProtocol.__cache[key] = item
return item;
def FetchOrCreateItem(summary):
"""Find an exisiting entry of this result, or create a new entry"""
key = ResultCommonEntryProtocolSet._CalculateKey(summary)
if key in ResultCommonEntryProtocolSet.__cache:
return ResultCommonEntryProtocolSet.__cache[key]
while True:
try:
sid = transaction.savepoint()
item, created = ResultCommonEntryProtocolSet.objects.get_or_create(
key=key)
if created:
item.protcol_result_summary.add(*summary)
transaction.savepoint_commit(sid)
except DatabaseError:
transaction.savepoint_rollback(sid)
time.sleep(0.1)
continue
except IntegrityError:
transaction.savepoint_rollback(sid)
time.sleep(0.1)
continue
except:
transaction.savepoint_rollback(sid)
transaction.rollback_unless_managed()
raise
break
ResultCommonEntryProtocolSet.__cache[key] = item
return item
def _saveData(request, fun, propList):
params = _retrievecmddict(request.POST.copy())
res = {}
try:
_checkkeys(params.keys(), propList)
res = fun(**params)
transaction.commit_unless_managed()
except TypeError as e:
activeinterlock_log.exception(e)
return HttpResponseBadRequest(HttpResponse(content=e))
except ValueError as e:
activeinterlock_log.exception(e)
return HttpResponseBadRequest(HttpResponse(content=e))
except MySQLError as e:
activeinterlock_log.exception(e)
return HttpResponseServerError(HttpResponse(content=e))
except TransactionManagementError as e:
activeinterlock_log.exception(e)
transaction.rollback_unless_managed()
return HttpResponseServerError(HttpResponse(content=e))
except Exception as e:
activeinterlock_log.exception(e)
raise e
return HttpResponse(json.dumps(res), content_type="application/json")
def load_customized_sql(app, created_models, db, verbosity=2, **kwargs):
"""Loads custom SQL from app/sql/post_syncdb-hook.<backend>.sql and
from app/sql/post_syncdb-hook.sql and send it to the database"""
app_dir = os.path.normpath(os.path.join(os.path.dirname(app.__file__),
'sql'))
custom_files = (os.path.join(app_dir, "post_syncdb-hook.%s.sql" %
settings.DATABASES[db]['ENGINE'].split('.')[-1]),
os.path.join(app_dir, "post_syncdb-hook.sql"))
for custom_file in custom_files:
if os.path.exists(custom_file):
if verbosity >= 2:
print("Loading customized SQL for %s using file %s" %
(app.__name__,
custom_file))
try:
with open(custom_file, 'U') as fp:
cursor = connections[db].cursor()
cursor.execute(fp.read().decode(settings.FILE_CHARSET))
except Exception as exc:
sys.stderr.write("Couldn't execute custom SQL for %s" %
app.__name__)
import traceback
traceback.print_exc(exc)
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
elif verbosity >= 2:
print("No customized SQL file %s" %
custom_file)
def has_pending_migrations():
"""
Returns whether any models need to be migrated via python manage.py migrate.
This will be the case if any migrations are present in apps, but not
in the database.
Shamelessly stolen from http://stackoverflow.com/questions/7089969/programmatically-check-whether-there-are-django-south-migrations-that-need-to-be
"""
apps = list(migration.all_migrations())
try:
applied_migrations = list(
MigrationHistory.objects.filter(
app_name__in=[app.app_label() for app in apps]))
except DatabaseError:
transaction.rollback_unless_managed()
return True # The table has not been created yet.
applied_migrations = [
'%s.%s' % (mi.app_name, mi.migration) for mi in applied_migrations
]
for app in apps:
for app_migration in app:
if app_migration.app_label() + "." + app_migration.name(
) not in applied_migrations:
return True
# No pending migrations.
return False
def run_sql_from_file(**kwargs):
app = get_app(kwargs.get('app'))
app_dir = os.path.normpath(
os.path.join(os.path.dirname(app.__file__), 'sql'))
if os.path.basename(app.__file__).find('__init__') == 0:
app_dir = os.path.normpath(
os.path.join(os.path.dirname(app.__file__), '..', 'sql'))
name = kwargs_parent.get('name', 'custom')
custom_files = [
os.path.join(app_dir,
"%s.%s.sql" % (name, settings.DATABASE_ENGINE)),
os.path.join(app_dir, "%s.sql" % name)
]
for custom_file in custom_files:
if os.path.exists(custom_file):
print "Loading SQL for %s from '%s'" % (
app.__name__, os.path.basename(custom_file))
sql = open(custom_file,
'U').read().decode("utf-8-sig").encode('utf-8')
cursor = connection.cursor()
try:
cursor.execute(sql)
except Exception, e:
print "Couldn't execute SQL for %s from %s" % (
app.__name__, os.path.basename(custom_file))
import traceback
traceback.print_exc()
transaction.rollback_unless_managed()
else:
transaction.commit_unless_managed()
def flush_all_tables(self):
"""
全テーブル内容を削除する。
ToDo: そもそもDjangoに備わっているJSON形式のレストアバッチコマンドを流用してもよい。
"""
#flush_cmd = flush.Command()
#flush_cmd.execute()
connection = connections[self._db]
style = no_style()
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError:
pass
sql_list = sql_flush(style, connection, only_django=True)
confirm = True
if ( confirm ):
try:
cursor = connection.cursor()
print "[Start Deleting]..."
for sql in sql_list:
print "%s... " % sql,
cursor.execute(sql)
print "Done."
print "[Done] delete tables completely!"
except Exception, e:
transaction.rollback_unless_managed(using=self._db)
raise CommandError("""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (connection.settings_dict['NAME'], e))
transaction.commit_unless_managed(using=self._db)
def insertDB(cursor, tablename, dic, update=False, privatekeys=None):
cmd = 'insert into %s (%s) value (%s) ' % (tablename,
','.join(dic.keys()), ','.join(('%s',) * len(dic)))
if update and privatekeys:
dic2 = copy.deepcopy(dic)
for k in privatekeys:
dic2.pop(k)
s = ', '.join(['%s' % k + '=%s' for k in dic2])
cmd += 'on duplicate key update %s' % s
dataexist = False
try:
if update:
cursor.execute(cmd, dic.values() + dic2.values())
else:
cursor.execute(cmd, dic.values())
cursor.connection.commit()
transaction.commit_unless_managed()
except (MySQLdb.IntegrityError, IntegrityError):
dataexist = True
cursor.connection.rollback()
transaction.rollback_unless_managed()
except Exception, e:
traceback.print_exc()
print 'insert db error. e:%s, dic: %s' % (e, str(dic))
cursor.connection.rollback()
transaction.rollback_unless_managed()
def findMpasInEezRaw(simplified=False):
for eez in Eez.objects.only('id'):
print 'Eez:', eez.eez
mpas = WdpaPolygon.objects.only('id').filter(geom__relate=(eez.geom, 'T********'))
cursor = connection.cursor()
try:
if (simplified):
try:
cursor.execute("SELECT mpa.id FROM %s as mpa, %s as eez WHERE eez.id = %%s AND ST_Relate(mpa.geom, eez.simple_geom, 'T********')" % (WdpaPolygon._meta.db_table, eez._meta.db_table), [eez.pk])
except:
transaction.rollback_unless_managed()
simplified = False
if (not simplified):
cursor.execute("SELECT mpa.id FROM %s as mpa, %s as eez WHERE eez.id = %%s AND ST_Relate(mpa.geom, eez.geom, 'T********')" % (WdpaPolygon._meta.db_table, eez._meta.db_table), [eez.pk])
for mpaid in [r[0] for r in cursor.fetchall()]:
mpa = WdpaPolygon.objects.get(pk=mpaid)
cursor = connection.cursor()
cursor.execute("SELECT ST_Area(ST_Intersection(mpa.geog, eez.geog), true) AS interarea FROM %s as mpa, %s as eez WHERE mpa.id = %%s AND eez.id = %%s" % (mpa._meta.db_table, eez._meta.db_table), [mpa.pk, eez.pk])
try:
area = cursor.fetchone()[0]
except:
area = None
mpamember = EezMembership(eez=eez, mpa=mpa, area_in_eez=area)
mpamember.save()
transaction.commit_unless_managed()
print mpa.name, ':', area/1000000, "km^2"
except:
transaction.rollback_unless_managed()
def handle_noargs(self, **options):
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
self.style = no_style()
sql_list = sql_cleardb(connection)
if options.get('interactive'):
confirm = raw_input("""
You have requested a database full clear.
This will IRREVERSIBLY DESTROY "%s" database
structure and data.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % (connection.settings_dict['NAME']),)
else:
confirm = 'yes'
if sql_list and confirm == 'yes':
try:
cursor = connection.cursor()
cursor.execute(sql_list)
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError("""Error: database couldn't be clear. Possible reasons:
* The database isn't running or isn't configured correctly.
* The SQL was invalid.
The full error: %s""" % (e,))
transaction.commit_unless_managed()
def updatelatticeinfo(params):
'''
'''
_checkkeys(params.keys(), ['function','name', 'version', 'branch', 'latticetype', 'description', 'creator'])
name = params['name']
version = params['version']
branch=params['branch']
latticetype=None
if params.has_key('latticetype'):
latticetype = params['latticetype']
latticetype = json.loads(latticetype)
if not isinstance(latticetype, dict):
raise TypeError("Lattice type data parameter format error.")
description = None
if params.has_key('description'):
description = params['description']
creator = None
if params.has_key('creator'):
creator = params['creator']
try:
result = latinst.updatelatticeinfo(name, version, branch, latticetype=latticetype, description=description, creator=creator)
transaction.commit_unless_managed()
except:
transaction.rollback_unless_managed()
raise
return {"result": result}
def _rollback_on_exception(**kwargs):
from django.db import transaction
for conn in connections:
try:
transaction.rollback_unless_managed(using=conn)
except DatabaseError:
pass
def handle_noargs(self, **options):
db = options.get('database')
connection = connections[db]
verbosity = int(options.get('verbosity'))
interactive = options.get('interactive')
self.style = no_style()
sql_list = sql_flush(self.style, connection, only_django=True)
if interactive:
confirm = raw_input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to the state it was in after syncdb.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
else:
confirm = 'yes'
if confirm == 'yes':
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed(using=db)
raise CommandError("""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (connection.settings_dict['NAME'], e))
transaction.commit_unless_managed(using=db)
def _drop_tables(connection, dbname, all_tables):
from django.db import transaction
tables = (
connection.introspection.table_names()
if all_tables
else connection.introspection.django_table_names(only_existing=True)
)
qn = connection.ops.quote_name
# use CASCADE with all backends except SQLite
sql_template = "DROP TABLE %s;" if "sqlite" in str(connection) else "DROP TABLE %s CASCADE;"
drop_table_sql = (sql_template % qn(table) for table in tables)
try:
cursor = connection.cursor()
for sql in drop_table_sql:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError(
"""Database '%s' couldn't be flushed.
%s occurred: %s
The full SQL: \n%s"""
% (dbname, e.__class__.__name__, e, "\n".join(drop_table_sql))
)
def sql_create_table(user_table):
model_class = opengis.create_model(user_table)
style = no_style()
tables = connection.introspection.table_names()
seen_models = connection.introspection.installed_models(tables)
pending_references = {}
sql, references = connection.creation.sql_create_model(model_class, style)
for refto, refs in references.items():
pending_references.setdefault(refto, []).extend(refs)
if refto in seen_models:
sql.extend(connection.creation.sql_for_pending_references(refto, style, pending_references))
sql.extend(connection.creation.sql_for_pending_references(model_class, style, pending_references))
cursor = connection.cursor()
for statement in sql:
cursor.execute(statement)
transaction.commit_unless_managed()
custom_sql = custom_sql_for_model(model_class, style)
if custom_sql:
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed()
else:
transaction.commit_unless_managed()
def _base_set(self, mode, key, value, timeout=None):
if timeout is None:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = datetime.now().replace(microsecond=0)
exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
encoded = base64.encodestring(pickle.dumps(value, 2)).strip()
cursor.execute("SELECT cache_key, expires FROM %s WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s WHERE cache_key = %%s" % table,
[encoded, connections[db].ops.value_to_db_datetime(exp), key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % table,
[key, encoded, connections[db].ops.value_to_db_datetime(exp)])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
transaction.rollback_unless_managed(using=db)
return False
else:
transaction.commit_unless_managed(using=db)
return True
def add_field(self, model, field_name):
"""
Generete ALTER SQL for given field and execute it
"""
opts = model._meta
field = opts.get_field(field_name)
qn = self.connection.ops.quote_name
col_type = field.db_type(self.connection)
field_output = [qn(field.column), col_type]
if not field.null:
field_output.append('NOT NULL')
field_output.append('DEFAULT %s' % qn(field.default))
sql_output = "ALTER TABLE %s ADD COLUMN %s;" % (
qn(opts.db_table), ' '.join(field_output))
# This is for debuging purpose only but for now I will not remove it
# Let's stay for informatvity
print "%s.%s.%s\n\t%s\n" % (opts.app_label, model.__name__, field.name, sql_output)
try:
cursor = self.connection.cursor()
cursor.execute(sql_output)
except Exception:
transaction.rollback_unless_managed()
raise
else:
transaction.commit_unless_managed()
def create_api_key_ignore_dberrors(*args, **kwargs):
try:
return create_api_key(*args, **kwargs)
except DatabaseError:
# no such table yet, first syncdb
from django.db import transaction
transaction.rollback_unless_managed()
def handle_app(self, app, **options):
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
app_name = app.__name__.split('.')[-2]
self.style = no_style()
sql_list = sql_reset(app, self.style, connection)
if options.get('interactive'):
confirm = raw_input("""
You have requested a database reset.
This will IRREVERSIBLY DESTROY any data for
the "%s" application in the database "%s".
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % (app_name, connection.settings_dict['NAME']))
else:
confirm = 'yes'
if confirm == 'yes':
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError("""Error: %s couldn't be reset. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlreset %s'. That's the SQL this command wasn't able to run.
The full error: %s""" % (app_name, app_name, e))
transaction.commit_unless_managed()
def create_default_locations(app, created_models, verbosity, **kwargs):
from geo.models import Location
from django.core.management import call_command
if Location in created_models and kwargs.get('interactive', True):
msg = "\nYou just installed the geo app, would you like to install the " \
"standard set of Locations? (yes/no): "
confirm = raw_input(msg)
while 1:
if confirm not in ('yes', 'no'):
confirm = raw_input('Please enter either "yes" or "no": ')
continue
if confirm == 'yes':
gunzip_location()
connection = connections[DEFAULT_DB_ALIAS]
cursor = connection.cursor()
custom_sql = custom_sql_for_model(Location, color_style(), connection)
if custom_sql:
if verbosity >= 1:
print "Installing custom SQL for geo.Location model"
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception, e:
sys.stderr.write("Failed to install custom SQL for geo.Location model: %s\n" % e)
transaction.rollback_unless_managed(using=DEFAULT_DB_ALIAS)
else:
transaction.commit_unless_managed(using=DEFAULT_DB_ALIAS)
else:
if verbosity >= 2:
print "No custom SQL for geo.Location model"
gzip_location()
break
def _updateData(request, fun, propList, customDict={}):
'''
Private template for update function
'''
params = _retrievecmddict(request.POST.copy())
res = {}
try:
_checkkeys(params.keys(), propList)
res = fun(**dict(customDict, **params))
transaction.commit_unless_managed()
except TypeError as e:
idods_log.exception(e)
return HttpResponseBadRequest(HttpResponse(content=e))
except ValueError as e:
idods_log.exception(e)
return HttpResponseBadRequest(HttpResponse(content=e))
except MySQLError as e:
idods_log.exception(e)
return HttpResponseServerError(HttpResponse(content=e))
except TransactionManagementError as e:
idods_log.exception(e)
transaction.rollback_unless_managed()
return HttpResponseServerError(HttpResponse(content=e))
except Exception as e:
idods_log.exception(e)
raise e
return HttpResponse(json.dumps(res), content_type="application/json")
def report_unexpected_exception(self, out, test, example, exc_info):
doctest.DocTestRunner.report_unexpected_exception(self, out, test,
example, exc_info)
# Rollback, in case of database errors. Otherwise they'd have
# side effects on other tests.
for conn in connections:
transaction.rollback_unless_managed(using=conn)
def handle_noargs(self, **options):
orig_load_initial_data = options.get('load_initial_data')
options['load_initial_data'] = False
super(Command, self).handle_noargs(**options)
db = options.get('database')
connection= connections[db]
# Create customs views for geojson_rest
if 'geojson_rest' in settings.INSTALLED_APPS:
app = models.get_app('geojson_rest')
app_models = models.get_models(app, include_auto_created=True)
tables = connection.introspection.table_names()
converter = connection.introspection.table_name_converter
custom_sql = sql_custom(models.get_app('geojson_rest'), no_style, connection)
#self.stdout.write(custom_sql)
if custom_sql:
cursor = connection.cursor()
try:
for sql in custom_sql:
cursor.execute(sql)
except Exception as e:
self.stdout.write(sql)
self.stderr.write("Failed to install custom SQL for geojson_rest: %s\n" % e)
transaction.rollback_unless_managed(using=db)
else:
transaction.commit_unless_managed(using=db)
# Load initial_data fixtures (unless that has been disabled)
if orig_load_initial_data:
call_command('loaddata', 'initial_data', verbosity=verbosity,
database=db, skip_validation=True)
def create_api_key_ignore_dberrors(*args, **kwargs):
try:
return create_api_key(*args, **kwargs)
except DatabaseError:
# no such table yet, first syncdb
from django.db import transaction
transaction.rollback_unless_managed()
def add_all_users(graph):
bind_namespaces(graph)
query = graph.query("""SELECT ?user ?email
WHERE {
?user perm:hasPermissionOver ?project .
?user foaf:mbox ?email .
?user rdf:type foaf:Agent
}""",
initNs=ns)
for q in query:
username = ""
# Remove username from customized uri
s = q[0].split("/")[-1]
username = s.split("'),")[0]
email = q[1].split("mailto:")[-1]
try:
u = User.objects.create_user(username, email,
DEFAULT_PASSWORD_STRING)
except IntegrityError:
transaction.rollback_unless_managed()
print "User '%s' already existed in the database, and was not created." % (
username)
else:
u.save()
print "User '%s' was created successfully." % (username)
def run_initial_sql(sender, **kwargs):
app_label = kwargs.get('app')
app_dir = os.path.normpath(os.path.join(os.path.dirname(
models.get_app(app_label).__file__), 'sql'))
if not os.path.exists(app_dir):
return
r = re.compile(r'^.*\.sql$')
sql_files = [os.path.join(app_dir, f)
for f in os.listdir(app_dir)
if r.match(f) is not None]
sql_files.sort()
cursor = connection.cursor()
for sql_file in sql_files:
try:
logger.info("Loading initial SQL data from '%s'" % sql_file)
f = open(sql_file)
sql = f.read()
f.close()
if not settings.TEST:
# Remove RAISE NOTICE (/!\ only one-liners)
sql = re.sub(r"\n.*RAISE NOTICE.*\n", "\n", sql)
# TODO: this is the ugliest driver hack ever
sql = sql.replace('%', '%%')
cursor.execute(sql)
except Exception as e:
print sql
logger.error("Failed to install custom SQL file '%s': %s\n" %
(sql_file, e))
traceback.print_exc()
transaction.rollback_unless_managed()
raise
else:
transaction.commit_unless_managed()
def add_all_users(graph):
bind_namespaces(graph)
query = graph.query("""SELECT ?user ?email
WHERE {
?user perm:hasPermissionOver ?project .
?user foaf:mbox ?email .
?user rdf:type foaf:Agent
}""", initNs = ns)
for q in query:
username = ""
# Remove username from customized uri
s = q[0].split("/")[-1]
username = s.split("'),")[0]
email = q[1].split("mailto:")[-1]
try:
u = User.objects.create_user(username, email, DEFAULT_PASSWORD_STRING)
except IntegrityError:
transaction.rollback_unless_managed()
print "User '%s' already existed in the database, and was not created."%(username)
else:
u.save()
print "User '%s' was created successfully."%(username)
def flush_all_tables(self):
"""
全テーブル内容を削除する。
ToDo: そもそもDjangoに備わっているJSON形式のレストアバッチコマンドを流用してもよい。
"""
#flush_cmd = flush.Command()
#flush_cmd.execute()
connection = connections[self._db]
style = no_style()
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError:
pass
sql_list = sql_flush(style, connection, only_django=True)
confirm = True
if (confirm):
try:
cursor = connection.cursor()
print "[Start Deleting]..."
for sql in sql_list:
print "%s... " % sql,
cursor.execute(sql)
print "Done."
print "[Done] delete tables completely!"
except Exception, e:
transaction.rollback_unless_managed(using=self._db)
raise CommandError(
"""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (connection.settings_dict['NAME'], e))
transaction.commit_unless_managed(using=self._db)
def _rollback_on_exception(**kwargs):
from django.db import transaction
for conn in connections:
try:
transaction.rollback_unless_managed(using=conn)
except DatabaseError:
pass
def run_sql(self, sql, **options):
"""
Parse through sql statements.
Execute each sql statement.
Commit in one transaction.
"""
# sql statement regex
statements = re.compile(r";[ \t]*$", re.M)
sql_list = []
# database information
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
for statement in statements.split(sql.decode(settings.FILE_CHARSET)):
# Remove any comments from the file
statement = re.sub(ur"--.*([\n\Z]|$)", "", statement)
if statement.strip():
sql_list.append(statement + u";")
try:
cursor = connection.cursor()
cursor.execute('SET FOREIGN_KEY_CHECKS = 0;')
for sql in sql_list:
cursor.execute(sql)
cursor.execute('SET FOREIGN_KEY_CHECKS = 1;')
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError("""
Error running SQL. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the database tables doesn't exist.
* The SQL was invalid.
The full error: %s""" % (e))
def _commit_on_success_unless_managed(*args, **kw):
try:
if transaction.is_managed():
forced_managed = False
else:
transaction.enter_transaction_management()
forced_managed = True
try:
res = func(*args, **kw)
except:
# All exceptions must be handled here (even string ones).
if transaction.is_dirty():
if forced_managed:
transaction.rollback()
else:
transaction.rollback_unless_managed()
raise
else:
if transaction.is_dirty():
if forced_managed:
transaction.commit()
else:
transaction.commit_unless_managed()
return res
finally:
if forced_managed:
transaction.leave_transaction_management()
def _base_set(self, mode, key, value, timeout=None):
if timeout is None:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = datetime.now().replace(microsecond=0)
exp = datetime.fromtimestamp(time.time() + timeout).replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
encoded = base64.encodestring(pickle.dumps(value, 2)).strip()
cursor.execute("SELECT cache_key, expires FROM %s WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute("UPDATE %s SET value = %%s, expires = %%s WHERE cache_key = %%s" % table,
[encoded, connections[db].ops.value_to_db_datetime(exp), key])
else:
cursor.execute("INSERT INTO %s (cache_key, value, expires) VALUES (%%s, %%s, %%s)" % table,
[key, encoded, connections[db].ops.value_to_db_datetime(exp)])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
transaction.rollback_unless_managed(using=db)
return False
else:
transaction.commit_unless_managed(using=db)
return True
def report_unexpected_exception(self, out, test, example, exc_info):
doctest.DocTestRunner.report_unexpected_exception(
self, out, test, example, exc_info)
# Rollback, in case of database errors. Otherwise they'd have
# side effects on other tests.
for conn in connections:
transaction.rollback_unless_managed(using=conn)
def run_sql(self, sql, **options):
"""
Parse through sql statements.
Execute each sql statement.
Commit in one transaction.
"""
# sql statement regex
statements = re.compile(r";[ \t]*$", re.M)
sql_list = []
# database information
using = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[using]
for statement in statements.split(sql.decode(settings.FILE_CHARSET)):
# Remove any comments from the file
statement = re.sub(ur"--.*([\n\Z]|$)", "", statement)
if statement.strip():
sql_list.append(statement + u";")
try:
cursor = connection.cursor()
cursor.execute('SET FOREIGN_KEY_CHECKS = 0;')
for sql in sql_list:
cursor.execute(sql)
cursor.execute('SET FOREIGN_KEY_CHECKS = 1;')
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError("""
Error running SQL. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the database tables doesn't exist.
* The SQL was invalid.
The full error: %s""" % (e))
def saveRawDataWS(request):
'''
Save raw data
'''
rawFile = request.FILES.getlist('file')[0]
res = {}
try:
res = idodsi.saveRawData(rawFile.read())
transaction.commit_unless_managed()
except TypeError as e:
idods_log.exception(e)
return HttpResponseBadRequest(HttpResponse(content=e))
except ValueError as e:
idods_log.exception(e)
return HttpResponseBadRequest(HttpResponse(content=e))
except MySQLError as e:
idods_log.exception(e)
return HttpResponseServerError(HttpResponse(content=e))
except TransactionManagementError as e:
idods_log.exception(e)
transaction.rollback_unless_managed()
return HttpResponseServerError(HttpResponse(content=e))
except Exception as e:
idods_log.exception(e)
raise e
return HttpResponse(json.dumps(res), content_type="application/json")
def handle_noargs(self, **options):
db = options.get('database')
connection= connections[db]
cursor = connection.cursor()
introspection = connection.introspection
# get the latest revision
cur_rev = schema_changes.pop()
# if cur_rev == 'master':
# print
apps = cur_rev[1]
for app in apps:
cur_app = app[0]
if cur_app in settings.INSTALLED_APPS:
print ('SQL to execute for application %s:' % cur_app)
# for sql in app[1]:
# print sql
for sql in app[1]:
try:
cursor.execute(sql)
except Exception as e:
# import ipdb; ipdb.set_trace()
# self.stdout.write(sql)
self.stderr.write(e.message)
self.stderr.write("SQL: {} for application {} is allready applied.\n".format(sql, cur_app))
transaction.rollback_unless_managed(using=db)
transaction.commit_unless_managed(using=db)
print('--------------------------------------------------------------------------------------')
def dell ( self, nameset, idsld ) :
cursor = connection.cursor()
try:
sld = self.get(id = idsld)
except:
return
namefile = sld.url
try:
cursor.execute( '''DELETE FROM tree_sld where id = %s;''', (idsld, ) )
transaction.commit_unless_managed()
except:
transaction.rollback_unless_managed()
return
if platform.system() == 'Linux':
waytofile = WAYTOSTYLE + nameset + "/" +namefile
if os.path.isfile(waytofile):
try:
os.remove(waytofile)
except:
return
else:
directr = directr.strip()
waytofile = WAYTOSTYLE + nameset + "\\" +namefile
if os.path.isfile(waytofile):
try:
os.remove(waytofile)
except:
return
def handle(self, filepath, **kwargs):
fp = csv.reader(open(filepath, "r"))
for row in fp:
try:
Thread.objects.create(thread_id=row[0], url=row[1])
except Exception, e:
transaction.rollback_unless_managed()
print e
def send_group_processors(group, **kwargs):
for inst in plugins.all():
try:
inst.post_process(group=group, **kwargs)
except:
transaction.rollback_unless_managed(using=group._state.db)
logger = logging.getLogger('sentry.plugins')
logger.exception('Error processing post_process() on %r', inst.__class__)
def handle_noargs(self, **options):
db = options.get('database', DEFAULT_DB_ALIAS)
connection = connections[db]
verbosity = int(options.get('verbosity', 1))
interactive = options.get('interactive')
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
import_module('.management', app_name)
except ImportError:
pass
sql_list = sql_flush(self.style, connection, only_django=True)
if interactive:
confirm = raw_input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to the state it was in after syncdb.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % connection.settings_dict['NAME'])
else:
confirm = 'yes'
if confirm == 'yes':
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed(using=db)
raise CommandError("""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (connection.settings_dict['NAME'], e))
transaction.commit_unless_managed(using=db)
# Emit the post sync signal. This allows individual
# applications to respond as if the database had been
# sync'd from scratch.
all_models = []
for app in models.get_apps():
all_models.extend([
m for m in models.get_models(app, include_auto_created=True)
if router.allow_syncdb(db, m)
])
emit_post_sync_signal(set(all_models), verbosity, interactive, db)
# Reinstall the initial_data fixture.
kwargs = options.copy()
kwargs['database'] = db
call_command('loaddata', 'initial_data', **kwargs)
def handle_noargs(self, **options):
from django.conf import settings
from django.db import connection, transaction, models
from django.dispatch import dispatcher
from django.core.management.sql import sql_flush, emit_post_sync_signal
verbosity = int(options.get('verbosity', 1))
interactive = options.get('interactive')
self.style = no_style()
# Import the 'management' module within each installed app, to register
# dispatcher events.
for app_name in settings.INSTALLED_APPS:
try:
__import__(app_name + '.management', {}, {}, [''])
except ImportError:
pass
sql_list = sql_flush(self.style, only_django=True)
if interactive:
confirm = input("""You have requested a flush of the database.
This will IRREVERSIBLY DESTROY all data currently in the %r database,
and return each table to the state it was in after syncdb.
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % settings.DATABASE_NAME)
else:
confirm = 'yes'
if confirm == 'yes':
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception as e:
transaction.rollback_unless_managed()
raise CommandError(
"""Database %s couldn't be flushed. Possible reasons:
* The database isn't running or isn't configured correctly.
* At least one of the expected database tables doesn't exist.
* The SQL was invalid.
Hint: Look at the output of 'django-admin.py sqlflush'. That's the SQL this command wasn't able to run.
The full error: %s""" % (settings.DATABASE_NAME, e))
transaction.commit_unless_managed()
# Emit the post sync signal. This allows individual
# applications to respond as if the database had been
# sync'd from scratch.
emit_post_sync_signal(models.get_models(), verbosity, interactive)
# Reinstall the initial_data fixture.
from django.core.management import call_command
call_command('loaddata', 'initial_data', **options)
else:
print("Flush cancelled.")
def cleanup(self):
cursor = self.connection_for_write().cursor()
try:
cursor.execute("DELETE FROM %s WHERE visible=%%s" % (
self.model._meta.db_table, ), (False, ))
except:
transaction.rollback_unless_managed()
else:
transaction.commit_unless_managed()
def bulk_update(objects):
try:
for o in objects:
o.save()
except Exception as err:
transaction.rollback_unless_managed()
raise err
else:
transaction.commit()
return objects
def reset_db():
using = DEFAULT_DB_ALIAS
connection = connections[using]
sql_list = sql_delete(elephantblog.models, no_style(), connection)
sql_list += sql_all(elephantblog.models, no_style(), connection)
try:
cursor = connection.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception, e:
transaction.rollback_unless_managed()
raise CommandError("Error: database couldn't be reset: %s" % e)
def insertDB(self, tablename, dic):
cmd = 'insert into %s (%s) value (%s)' % (tablename, ','.join(
dic.keys()), ','.join(('%s', ) * len(dic)))
dataexist = False
try:
self.execute_sql(cmd, dic.values())
self.cursor.connection.commit()
transaction.commit_unless_managed()
except (MySQLdb.IntegrityError, IntegrityError), e:
dataexist = True
self.cursor.connection.rollback()
transaction.rollback_unless_managed()
def _base_set(self, mode, key, value, timeout=None):
if timeout is None:
timeout = self.default_timeout
db = router.db_for_write(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
cursor = connections[db].cursor()
cursor.execute("SELECT COUNT(*) FROM %s" % table)
num = cursor.fetchone()[0]
now = timezone.now()
now = now.replace(microsecond=0)
if settings.USE_TZ:
exp = datetime.utcfromtimestamp(time.time() + timeout)
else:
exp = datetime.fromtimestamp(time.time() + timeout)
exp = exp.replace(microsecond=0)
if num > self._max_entries:
self._cull(db, cursor, now)
pickled = pickle.dumps(value, pickle.HIGHEST_PROTOCOL)
b64encoded = base64.b64encode(pickled)
# The DB column is expecting a string, so make sure the value is a
# string, not bytes. Refs #19274.
if six.PY3:
b64encoded = b64encoded.decode('latin1')
cursor.execute(
"SELECT cache_key, expires FROM %s "
"WHERE cache_key = %%s" % table, [key])
try:
result = cursor.fetchone()
if result and (mode == 'set' or
(mode == 'add' and result[1] < now)):
cursor.execute(
"UPDATE %s SET value = %%s, expires = %%s "
"WHERE cache_key = %%s" % table, [
b64encoded,
connections[db].ops.value_to_db_datetime(exp), key
])
else:
cursor.execute(
"INSERT INTO %s (cache_key, value, expires) "
"VALUES (%%s, %%s, %%s)" % table, [
key, b64encoded,
connections[db].ops.value_to_db_datetime(exp)
])
except DatabaseError:
# To be threadsafe, updates/inserts are allowed to fail silently
transaction.rollback_unless_managed(using=db)
return False
else:
transaction.commit_unless_managed(using=db)
return True
def _inner(*args, **kwargs):
_max_retries = kwargs.pop("exception_retry_count", max_retries)
for retries in count(0):
try:
return fun(*args, **kwargs)
except Exception: # pragma: no cover
# Depending on the database backend used we can experience
# various exceptions. E.g. psycopg2 raises an exception
# if some operation breaks the transaction, so saving
# the task result won't be possible until we rollback
# the transaction.
if retries >= _max_retries:
raise
transaction.rollback_unless_managed()
def _autocommit(*args, **kw):
try:
transaction.enter_transaction_management()
transaction.managed(False)
try:
result = func(*args, **kw)
except:
transaction.rollback_unless_managed()
raise
else:
transaction.commit_unless_managed()
return result
finally:
transaction.leave_transaction_management()
def _reset_sequences(self, db_name):
conn = connections[db_name]
if conn.features.supports_sequence_reset:
sql_list = \
conn.ops.sequence_reset_by_name_sql(no_style(),
conn.introspection.sequence_list())
if sql_list:
try:
cursor = conn.cursor()
for sql in sql_list:
cursor.execute(sql)
except Exception:
transaction.rollback_unless_managed(using=db_name)
raise
transaction.commit_unless_managed(using=db_name)
def safe_execute(func, *args, **kwargs):
try:
result = func(*args, **kwargs)
except Exception, e:
transaction.rollback_unless_managed()
if hasattr(func, 'im_class'):
cls = func.im_class
else:
cls = func.__class__
logger = logging.getLogger('sentry.plugins')
logger.error('Error processing %r on %r: %s', func.__name__, cls.__name__, e, extra={
'func_module': cls.__module__,
'func_args': args,
'func_kwargs': kwargs,
}, exc_info=True)
def create_sort_index(cls, sender, db, created_models, **kwargs):
# This is only supported in postgres
engine = get_db_engine()
if not engine.startswith('postgresql'):
return
if cls not in created_models:
return
from django.db import connections
try:
cursor = connections[db].cursor()
cursor.execute("create index sentry_groupedmessage_score on sentry_groupedmessage ((%s))" % (cls.get_score_clause(),))
cursor.close()
except:
transaction.rollback_unless_managed()
def handle_label(self, tablename, **options):
db = options.get('database')
cache = BaseDatabaseCache(tablename, {})
if not router.allow_syncdb(db, cache.cache_model_class):
return
connection = connections[db]
fields = (
# "key" is a reserved word in MySQL, so use "cache_key" instead.
models.CharField(name='cache_key',
max_length=255,
unique=True,
primary_key=True),
models.TextField(name='value'),
models.DateTimeField(name='expires', db_index=True),
)
table_output = []
index_output = []
qn = connection.ops.quote_name
for f in fields:
field_output = [qn(f.name), f.db_type(connection=connection)]
field_output.append("%sNULL" % (not f.null and "NOT " or ""))
if f.primary_key:
field_output.append("PRIMARY KEY")
elif f.unique:
field_output.append("UNIQUE")
if f.db_index:
unique = f.unique and "UNIQUE " or ""
index_output.append("CREATE %sINDEX %s ON %s (%s);" % \
(unique, qn('%s_%s' % (tablename, f.name)), qn(tablename),
qn(f.name)))
table_output.append(" ".join(field_output))
full_statement = ["CREATE TABLE %s (" % qn(tablename)]
for i, line in enumerate(table_output):
full_statement.append(
' %s%s' % (line, i < len(table_output) - 1 and ',' or ''))
full_statement.append(');')
curs = connection.cursor()
try:
curs.execute("\n".join(full_statement))
except DatabaseError as e:
transaction.rollback_unless_managed(using=db)
raise CommandError(
"Cache table '%s' could not be created.\nThe error was: %s." %
(tablename, e))
for statement in index_output:
curs.execute(statement)
transaction.commit_unless_managed(using=db)
def test_insertdb():
cursor = get_connect()
cmd = 'insert into word_dict2 (doc_w,title_w,p) value (%s,%s,%s) ' + \
'on duplicate key update p=%s'
for i in range(1000):
try:
cursor.execute(cmd, [i, i, i * .1, i * .1])
cursor.connection.commit()
transaction.commit_unless_managed()
except (MySQLdb.IntegrityError, IntegrityError):
cursor.connection.rollback()
transaction.rollback_unless_managed()
except Exception, e:
print e
traceback.print_exc()
cursor.connection.rollback()
transaction.rollback_unless_managed()
def run_if_needed(self,
sql,
ok_errors=("already exists", "does not exist")):
"""Run the sql, ignoring any errors that contain an ok_error
This *will* rollback the transaction on error to avoid the 'error state' error"""
transaction.commit_unless_managed()
try:
cursor = connection.cursor()
cursor.execute(sql)
transaction.commit_unless_managed()
except DatabaseError, e:
log.debug(str(e))
for ok_error in ok_errors:
if ok_error in str(e):
transaction.rollback_unless_managed()
return
raise
def handle(self, *model_labels, **options):
if settings.DATABASES:
db_engine = settings.DATABASES[settings.DATABASES.keys()[0]]['ENGINE'].split('.')[-1]
else:
db_engine = settings.DATABASE_ENGINE
if db_engine != 'sqlite3' or not settings.DEBUG:
raise CommandError('simplemigration should only be used on sqlite databases with DEBUG = True (not for production use).')
for label in model_labels:
m = models.get_model(*label.split('.'))
temp_name = '_' + m._meta.db_table
try:
# rename table, then recreate
cursor = connection.cursor()
sql = 'ALTER TABLE %s RENAME TO %s' % (m._meta.db_table, temp_name)
cursor.execute(sql)
print sql
call_command('syncdb')
cursor = connection.cursor()
old_fields = [f[0] for f in connection.introspection.get_table_description(cursor, temp_name)]
new_fields = [f[0] for f in connection.introspection.get_table_description(cursor, m._meta.db_table)]
insert_fields = [f if f in old_fields else '\'\'' for f in new_fields]
# insert data from temp table into new
sql = 'INSERT INTO %s SELECT %s FROM %s' % (m._meta.db_table, ', '.join(insert_fields), temp_name)
cursor.execute(sql)
print sql
# drop temporary table
sql = 'DROP TABLE %s' % (temp_name)
cursor.execute(sql)
print sql
except Exception, e:
print 'Last SQL statement: ' + sql
transaction.rollback_unless_managed()
raise
else:
transaction.commit_unless_managed()
