def _order_in_memory(self, lhs, rhs):
for order in self.compiler._get_ordering():
if LOOKUP_SEP in order:
raise DatabaseError("JOINs in ordering not supported (%s)" %
order)
if order == '?':
result = random.choice([1, 0, -1])
else:
column = order.lstrip('-')
result = cmp(lhs.get(column), rhs.get(column))
if order.startswith('-'):
result *= -1
if result != 0:
return result
return 0
def validate_thread_sharing(self):
"""
Validate that the connection isn't accessed by another thread than the
one which originally created it, unless the connection was explicitly
authorized to be shared between threads (via the `inc_thread_sharing()`
method). Raise an exception if the validation fails.
"""
if not (self.allow_thread_sharing or self._thread_ident == _thread.get_ident()):
raise DatabaseError(
"DatabaseWrapper objects created in a "
"thread can only be used in that same thread. The object "
"with alias '%s' was created in thread id %s and this is "
"thread id %s."
% (self.alias, self._thread_ident, _thread.get_ident())
)
def _value_for_db_key(self, value, field_kind):
"""
Converts value to be used as a key to an acceptable type.
On default we do no encoding, only allowing key values directly
acceptable by the database for its key type (if any).
The conversion has to be reversible given the field type,
encoding should preserve comparisons.
Use this to expand the set of fields that can be used as
primary keys, return value suitable for a key rather than
a key itself.
"""
raise DatabaseError("%s may not be used as primary key field." %
field_kind)
def check_query(self):
"""
Checks if the current query is supported by the database.
In general, we expect queries requiring JOINs (many-to-many
relations, abstract model bases, or model spanning filtering),
using DISTINCT (through `QuerySet.distinct()`, which is not
required in most situations) or using the SQL-specific
`QuerySet.extra()` to not work with nonrel back-ends.
"""
if (len([
a for a in self.query.alias_map if self.query.alias_refcount[a]
]) > 1 or self.query.distinct or self.query.extra
or self.query.having):
raise DatabaseError("This query is not supported by the database.")
def _value_for_db(self, value, field, field_kind, db_type, lookup):
"""
Allows parent to handle nonrel fields, convert AutoField
keys to ObjectIds and date and times to datetimes.
Let everything else pass to PyMongo -- when the value is used
the driver will raise an exception if it got anything
unacceptable.
"""
if value is None:
return None
# Parent can handle iterable fields and Django wrappers.
value = super(DatabaseOperations, self)._value_for_db(
value, field, field_kind, db_type, lookup)
# Convert decimals to strings preserving order.
if field_kind == 'DecimalField':
value = decimal_to_string(
value, field.max_digits, field.decimal_places)
# Anything with the "key" db_type is converted to an ObjectId.
if db_type == 'key':
try:
return ObjectId(value)
# Provide a better message for invalid IDs.
except (TypeError, InvalidId):
if isinstance(value, (str, unicode)) and len(value) > 13:
value = value[:10] + '...'
msg = "AutoField (default primary key) values must be " \
"strings representing an ObjectId on MongoDB (got " \
"%r instead)." % value
if field.model._meta.db_table == 'django_site':
# Also provide some useful tips for (very common) issues
# with settings.SITE_ID.
msg += " Please make sure your SITE_ID contains a " \
"valid ObjectId string."
raise DatabaseError(msg)
# PyMongo can only process datatimes?
elif db_type == 'date':
return datetime.datetime(value.year, value.month, value.day)
elif db_type == 'time':
return datetime.datetime(1, 1, 1, value.hour, value.minute,
value.second, value.microsecond)
return value
def execute_sql(self, result_type):
data = {}
for field, model, value in self.query.values:
assert field is not None
if value is None and not field.null:
raise DatabaseError("It is not possible to set None to a ",
"non-nullable field %s!" % field.name)
db_type = field.db_type(connection=self.connection)
value = self.convert_value_for_db(db_type, value)
data[field.column] = value
return self.update(data)
def validate_thread_sharing(self):
"""
检测数据库并没有被其他的线程连接, 除非已经设置为所有线程可以共享
Validates that the connection isn't accessed by another thread than the
one which originally created it, unless the connection was explicitly
authorized to be shared between threads (via the `allow_thread_sharing`
property). Raises an exception if the validation fails.
"""
if (not self.allow_thread_sharing
and self._thread_ident != thread.get_ident()):
raise DatabaseError("DatabaseWrapper objects created in a "
"thread can only be used in that same thread. The object "
"with alias '%s' was created in thread id %s and this is "
"thread id %s."
% (self.alias, self._thread_ident, thread.get_ident()))
def test_create_export_task_record(self, mock_export_task):
from ..export_tasks import TaskStates
task_name = "TaskName"
export_provider_task_name = "ExportProviderTaskName"
worker = "Worker"
expected_result = MagicMock(display=False)
mock_export_task.objects.create.return_value = expected_result
task_result = create_export_task_record(
task_name=task_name,
export_provider_task=export_provider_task_name,
worker=worker,
display=False)
self.assertEquals(task_result, expected_result)
mock_export_task.objects.create.assert_called_with(
export_provider_task=export_provider_task_name,
status=TaskStates.PENDING.value,
name=task_name,
worker=worker,
display=False)
expected_result = MagicMock(display=True)
mock_export_task.objects.create.return_value = expected_result
task_result = create_export_task_record(
task_name=task_name,
export_provider_task=export_provider_task_name,
worker=worker,
display=True)
self.assertEquals(task_result, expected_result)
mock_export_task.objects.create.assert_called_with(
export_provider_task=export_provider_task_name,
status=TaskStates.PENDING.value,
name=task_name,
worker=worker,
display=True)
mock_export_task.objects.create.side_effect = DatabaseError(
"SomeError")
with self.assertRaises(DatabaseError):
create_export_task_record(
task_name=task_name,
export_provider_task=export_provider_task_name,
worker=worker,
display=True)
def fetch(self, low_mark, high_mark):
if self.root_predicate == None:
raise DatabaseError('No root query node')
try:
if high_mark is not None and high_mark <= low_mark:
return
results = self._get_query_results()
if low_mark is not None or high_mark is not None:
results = results[low_mark:high_mark]
except Exception, e:
# FIXME: Can get rid of this exception handling code eventually,
# but it's useful for debugging for now.
#traceback.print_exc()
raise e
def make_result(self, entity, fields):
result = []
for field in fields:
value = entity.get(field.column, NOT_PROVIDED)
if value is NOT_PROVIDED:
value = field.get_default()
else:
value = self.convert_value_from_db(
field.db_type(connection=self.connection), value)
if not field.null and value is None:
raise DatabaseError("Non-nullable field %s can't be None!" %
field.name)
result.append(value)
return result
def test_auditable_model_delete_with_database_error(self):
luy = LearningUnitYearFactory()
geys = [GroupElementYearFactory(child_leaf=luy) for i in range(20)]
result = None
with patch.object(AuditableModel, 'save') as mock_method:
mock_method.side_effect = DatabaseError("test error")
try:
luy.delete()
except DatabaseError as e:
result = e
self.assertIsInstance(result, DatabaseError)
self.is_existing_for_orm(luy)
for gey in geys:
self.is_existing_for_orm(gey)
class CassandraConnection(object):
def __init__(self, host, port, keyspace, username, password, pool_size=5):
self.host = host
self.port = port
self.keyspace = keyspace
self.username = username
self.password = password
self.pool_size = pool_size
self.logged_in = False
self.pool = None
def commit(self):
pass
def open(self):
server_list = [self.host + ":" + self.port]
connected = False
try:
if self.username and self.password:
credentials = {
'username': self.username,
'password': self.password
}
self.pool = ConnectionPool(keyspace=self.keyspace,
server_list=server_list,
credentials=credentials,
pool_size=self.pool_size)
if self.pool:
self.logged_in = True
connected = True
else:
self.pool = ConnectionPool(keyspace=self.keyspace,
server_list=server_list,
pool_size=self.pool_size)
if self.pool:
connected = True
except Exception, e:
pass
if not connected:
raise DatabaseError('Error connecting to keyspace: %s; %s' %
(self.keyspace, str(e)))
def login(self):
# TODO: This user/password auth code hasn't been tested
if not self.logged_in:
if self.user:
try:
if self.client:
credentials = {'username': self.user, 'password': self.password}
self.client.login(AuthenticationRequest(credentials))
self.logged_in = True
except Exception, e:
# In this case we won't have set logged_in to true, so we'll throw the
# exception below where it also handles the case that self.client
# is not valid yet.
pass
if not self.logged_in:
raise DatabaseError('Error logging in to keyspace: %s; %s' % (self.keyspace, str(e)))
else:
self.logged_in = True
def _create_trigger(self, field):
from django.db.utils import DatabaseError
cursor = self.connection.cursor()
opts = field.model._meta
trigger_name = get_trigger_name(field, opts)
stms = self.sql.split('##')
for template in stms:
stm = template.format(trigger_name=trigger_name,
opts=opts,
field=field)
try:
cursor.execute(stm)
except BaseException as exc:
raise DatabaseError(exc)
return trigger_name
def _get_s3_bucket(cls, bucket_name, assert_versioning_enabled):
try:
connection = cls._get_s3_connection()
if not connection.lookup(bucket_name):
connection.create_bucket(bucket_name)
bucket = connection.get_bucket(bucket_name)
if assert_versioning_enabled:
cls._assert_bucket_has_versioning_enabled(bucket)
return bucket
except boto_exceptions.S3ResponseError as e:
raise DatabaseError(
"Error retrieving bucket from S3.\nBucket: {}\nBoto Exception:\n{}".format(
bucket_name, e
)
)
def test_scale_post_steps_database_error(self, mock_env_vars, mock_db,
mock_sys_exit):
"""Tests executing scale_post_steps when a database error occurs."""
# Set up mocks
def get_env_vars(name, *args, **kwargs):
return str(self.job.id) if name == 'SCALE_JOB_ID' else str(
self.job_exe.exe_num)
mock_env_vars.side_effect = get_env_vars
mock_db.side_effect = DatabaseError()
# Call method to test
cmd = PostCommand()
cmd.run_from_argv(['manage.py', 'scale_post_steps'])
# Check results
mock_sys_exit.assert_called_with(ScaleDatabaseError().exit_code)
def test_raises_if_save_raises_database_error(self):
mock_pod_form = Mock()
self.mock_PodForm.return_value = mock_pod_form
mock_pod_form.errors = {}
mock_pod_form.is_valid = Mock()
mock_pod_form.is_valid.return_value = True
mock_pod_form.save = Mock()
mock_pod_form.save.side_effect = DatabaseError("broken transaction")
node = factory.make_Node_with_Interface_on_Subnet(
status=NODE_STATUS.DEPLOYING,
agent_name="maas-kvm-pod",
install_kvm=True,
)
factory.make_StaticIPAddress(interface=node.boot_interface)
NodeMetadata.objects.create(node=node,
key="virsh_password",
value="xyz123")
self.assertRaises(DatabaseError, _create_pod_for_deployment, node)
def _get_s3_connection(cls):
s3_connection = getattr(thread_safe_connection_cache, 's3_connection', None)
if s3_connection:
return s3_connection
aws_access_key_id = getattr(settings, 'AWS_S3_FILES_ACCESS_KEY_ID', None)
aws_secret_access_key = getattr(settings, 'AWS_S3_FILES_SECRET_ACCESS_KEY', None)
assert aws_access_key_id and aws_secret_access_key, \
"Please assign values for AWS_S3_FILES_ACCESS_KEY_ID and AWS_S3_FILES_SECRET_ACCESS_KEY in settings"
try:
connection = S3Connection(aws_access_key_id, aws_secret_access_key) # TODO: We may want to move to a connection pool
setattr(thread_safe_connection_cache, 's3_connection', connection)
except boto_exceptions.BotoClientError as e:
raise DatabaseError("Unable to connect to S3.\nBoto Exception:\n{}".format(e))
return thread_safe_connection_cache.s3_connection
def insert(self, docs, return_id=False):
"""
Stores a document using field columns as element names, except
for the primary key field for which "_id" is used.
If just a {pk_field: None} mapping is given a new empty
document is created, otherwise value for a primary key may not
be None.
"""
for doc in docs:
try:
doc['_id'] = doc.pop(self.query.get_meta().pk.column)
except KeyError:
pass
if doc.get('_id', NOT_PROVIDED) is None:
if len(doc) == 1:
# insert with empty model
doc.clear()
else:
raise DatabaseError(
"Can't save entity with _id set to None")
collection = self.get_collection()
options = self.connection.operation_flags.get('save', {})
# Solution to fix save is deprecated.
# if id is in the doc, the data needs to be updated else data needs to be inserted
try:
doc['_id']
is_update = True
update_spec = {'$set': doc}
except KeyError:
is_update = False
if return_id:
if is_update:
return collection.update_one({'_id': doc['_id']}, update_spec,
True)
else:
return collection.insert_one(doc)
else:
if is_update:
#return collection.save(doc, **options)
collection.update_one({'_id': doc['_id']}, update_spec, True)
else:
collection.insert_one(doc)
def _create_trigger(self, field):
from django.db.utils import DatabaseError
opts = field.model._meta
trigger_name = get_trigger_name(field, opts)
stm = self.sql.format(trigger_name=trigger_name,
opts=opts,
field=field)
self.connection.drop_trigger('{}_i'.format(trigger_name))
self.connection.drop_trigger('{}_u'.format(trigger_name))
try:
self.connection.cursor().execute(stm)
self._triggers[field] = trigger_name
except BaseException as exc: # pragma: no cover
raise DatabaseError(exc)
return trigger_name
def _add_filter(self, column, op, db_type, value):
for query in self.gae_query:
key = '%s %s' % (column, op)
value = self.convert_value_for_db(db_type, value)
if isinstance(value, Text):
raise DatabaseError('TextField is not indexed, by default, '
"so you can't filter on it. Please add "
'an index definition for the column %s '
'on the model %s.%s as described here:\n'
'http://www.allbuttonspressed.com/blog/django/2010/07/Managing-per-field-indexes-on-App-Engine'
% (column, self.query.model.__module__, self.query.model.__name__))
if key in query:
existing_value = query[key]
if isinstance(existing_value, list):
existing_value.append(value)
else:
query[key] = [existing_value, value]
else:
query[key] = value
def check_query(self):
"""
Checks if the current query is supported by the database.
In general, we expect queries requiring JOINs (many-to-many
relations, abstract model bases, or model spanning filtering),
using DISTINCT (through `QuerySet.distinct()`, which is not
required in most situations) or using the SQL-specific
`QuerySet.extra()` to not work with nonrel back-ends.
"""
if hasattr(self.query, 'is_empty') and self.query.is_empty():
raise EmptyResultSet()
if (len([
a for a in self.query.alias_map if self.query.alias_refcount[a]
]) > 1 or self.query.distinct
or self.query.extra): # or self.having -- Not quite working.
# having is no longer part of the query as of 1.9; It moved to the compiler
# https://github.com/django/django/commit/afe0bb7b13bb8dc4370f32225238012c873b0ee3
raise DatabaseError("This query is not supported by the database.")
def sql_create_model(self, model, style, known_models=set()):
db_connection = self.connection.db_connection
keyspace = self.connection.settings_dict['NAME']
opts = model._meta
column_metadata = []
# Browsing through fields to find indexed fields
for field in opts.local_fields:
if field.db_index:
column_name = str(
field.db_column if field.db_column else field.column)
column_def = ColumnDef(name=column_name,
validation_class='BytesType',
index_type=IndexType.KEYS)
column_metadata.append(column_def)
cfdef_settings = self.connection.column_family_def_defaults.copy()
if hasattr(model, 'CassandraSettings') and \
hasattr(model.CassandraSettings, 'COLUMN_FAMILY_DEF_SETTINGS'):
cfdef_overrides = model.CassandraSettings.COLUMN_FAMILY_DEF_SETTINGS
if type(cfdef_overrides) is not dict:
raise DatabaseError(
'The value of COLUMN_FAMILY_DEF_SETTINGS in the '
'CassandraSettings class must be a dictionary of the optional '
'settings to use when creating the column family.')
cfdef_settings.update(cfdef_overrides)
cfdef_settings['keyspace'] = keyspace
if not cfdef_settings.get('name'):
cfdef_settings['name'] = opts.db_table
if not cfdef_settings.get('comparator_type'):
cfdef_settings['comparator_type'] = 'UTF8Type'
cfdef_settings['column_metadata'] = column_metadata
column_family_def = CfDef(**cfdef_settings)
db_connection.get_client().system_add_column_family(column_family_def)
return [], {}
def connect_to_db():
try:
keyspace = DATABASES['cassandra']['NAME']
username = DATABASES['cassandra']['USER']
password = DATABASES['cassandra']['PASSWORD']
host = DATABASES['cassandra']['HOST']
port = DATABASES['cassandra']['PORT']
pool_size = DATABASES['cassandra']['POOL_SIZE']
con = CassandraConnection(host,
port,
keyspace,
username,
password,
pool_size=pool_size)
return con
except Exception, e:
raise DatabaseError(
'Error connecting to database. Incorrect configuration in settings file: %s'
% str(e))
def combine_duration_expression(self, connector, sub_expressions):
"""Combine duration expressions into single one using connector.
:type connector: str
:param connector: A type of connector operation.
:type sub_expressions: list
:param sub_expressions: A list of expressions to combine.
:raises: :class:`~django.db.utils.DatabaseError`
:rtype: str
:return: A SQL statement for combining.
"""
if connector == "+":
return "TIMESTAMP_ADD(" + ", ".join(sub_expressions) + ")"
elif connector == "-":
return "TIMESTAMP_SUB(" + ", ".join(sub_expressions) + ")"
else:
raise DatabaseError("Invalid connector for timedelta: %s." %
connector)
def _save(self, data, return_id=False):
collection = self.get_collection()
options = self.connection.operation_flags.get('save', {})
# Check for fields set to None
if None in data.values():
# Check if we have null + unique fields set to None. If so delete
# those from data, so that the sparse index works
fields = self.get_fields()
for field in fields:
if field.null and field.unique and data[field.column] is None:
del data[field.column]
if data.get('_id', NOT_PROVIDED) is None:
if len(data) == 1:
# insert with empty model
data = {}
else:
raise DatabaseError("Can't save entity with _id set to None")
primary_key = collection.save(data, **options)
if return_id:
return unicode(primary_key)
def connect(self):
super(DatabaseWrapper, self).connect()
if not (self.connection is not None
and self.django_pool_params.django_pre_ping):
return
is_usable = False
ex_message = None
ex_default = 'unable to connect to the database'
try:
is_usable = self.is_usable() # ping
except Exception as ex:
ex_message = str(ex)
finally:
if not is_usable:
try:
self.errors_occurred = True
self.close()
finally:
raise DatabaseError(ex_message or ex_default)
def get_fields(self):
"""
Returns the fields which should get loaded from the backend by self.query
"""
# We only set this up here because
# related_select_fields isn't populated until
# execute_sql() has been called.
if self.query.select_fields:
fields = self.query.select_fields + self.query.related_select_fields
else:
fields = self.query.model._meta.fields
# If the field was deferred, exclude it from being passed
# into `resolve_columns` because it wasn't selected.
only_load = self.deferred_to_columns()
if only_load:
db_table = self.query.model._meta.db_table
fields = [f for f in fields if db_table in only_load and
f.column in only_load[db_table]]
for field in fields:
if field.model._meta != self.query.model._meta:
raise DatabaseError('Multi-table inheritance is not supported '
'by non-relational DBs.')
return fields
def table_names(self):
cursor = connection.cursor()
try:
# mysql
campus_tables = "SELECT table_name \
FROM INFORMATION_SCHEMA.TABLES \
WHERE table_schema='%s' \
AND (LOCATE('campus_', table_name) = 1 OR LOCATE('south_', table_name) = 1)" % settings.DATABASES['default']['NAME']
cursor.execute(campus_tables)
return cursor
except DatabaseError:
pass
try:
# sqlite
campus_tables = "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND (name LIKE 'campus_%%' OR name LIKE 'south_%%')"
cursor.execute(campus_tables)
return cursor
except DatabaseError:
pass
raise DatabaseError('Can not read table names from the database')
def update(self, values):
"""
Changes an entity that already exists in the database.
:param values: A list of (field, new-value) pairs.
"""
pool = self.connection
column_family_name = get_column_family()
col_fam = CF(pool, column_family_name)
pk_column = get_pk_column()
pk_index = -1
fields = self.get_fields()
for index in range(len(fields)):
if fields[index].column == pk_column:
pk_index = index
break
if pk_index == -1:
raise DatabaseError('Invalid primary key column.')
b = col_fam.batch(
write_consistency_level=self.connection.write_consistency_level)
row_count = 0
for result in self.results_iter():
row_count += 1
key = result[pk_index]
for k, v in values.items():
b.insert(k, v)
b.send()
return row_count
def _execute_raise_database_already_exists(self, cursor, parameters, keepdb=False):
error = DatabaseError('database %s already exists' % parameters['dbname'])
error.pgcode = errorcodes.DUPLICATE_DATABASE
raise DatabaseError() from error
def _execute_raise_permission_denied(self, cursor, parameters, keepdb=False):
error = DatabaseError('permission denied to create database')
error.pgcode = errorcodes.INSUFFICIENT_PRIVILEGE
raise DatabaseError() from error
