def update_index(self, index, documents):
model = self.index_model_factory(index)
with transaction.atomic():
# We won't allow records who utilize unused fields
# Delete them so are always getting unique query results.
if index._unused_fields:
q = Q()
for field_name in six.iterkeys(index._unused_fields):
q |= Q(**{'%s__isnull' % field_name: False})
invalid = model.objects.filter(q, document__in=documents)
invalid.delete()
# We also won't allow records who have haven't all
# _unique_together fields populated. Clean them up.
q = Q()
for field_name in six.iterkeys(index._unique_together):
q |= Q(**{'%s__isnull' % field_name: True})
stale = model.objects.filter(q, document__in=documents)
stale.delete()
for document in documents:
records = index.build_records(document)
for record in records:
model.objects.update_or_create(**dict(defaults=record, **{
field_name: record[field_name]
for field_name in six.iterkeys(index._unique_together)
}))
def test_get_fields_with_include_all_and_exclude(self):
expected = six.iterkeys(UserSerializer().get_all_fields())
serializer = UserSerializer(
include_fields='*',
exclude_fields=['id']
)
self.assertEqual(list(six.iterkeys(serializer.fields)), list(expected))
def __all__(self):
names = set()
for name in six.iterkeys(self._attrs):
names.add(name)
for name in six.iterkeys(self._base_attrs):
names.add(name)
return list(names)
def sync_index(self, index, documents, full=False):
model = index._model
related_models = index._related_models
all_pks = set()
# Sync documents
for document in documents:
records = index.build_records(document)
pks = set()
for record in records:
# Filter out related fields, we'll handle
# these later on
defaults = {
field_name: record[field_name]
for field_name in six.iterkeys(index.fields)
if field_name not in related_models
}
# Construct the unique query kwargs
unique_together = {
field_name: record[field_name] for field_name in six.iterkeys(index._unique_together)
}
# Check for existing records
existing = model.objects.filter(**unique_together)
if existing.count() > 1:
# Some records have become stale
# delete all but one
model.objects.filter(**unique_together).exclude(pk=existing.first().pk).delete()
with transaction.atomic():
# Update or create index
obj, created = model.objects.update_or_create(**dict(defaults=defaults, **unique_together))
pks.add(obj.pk)
# Update or create related indexes
for field_name, related_model in six.iteritems(related_models):
values = record[field_name] or []
# Delete stale records
related_model.objects.filter(index=obj).exclude(value__in=values).delete()
# Ensure related records
for value in values:
related_model.objects.get_or_create(index=obj, value=value)
# Check for stale records
stale = model.objects.filter(document=document).exclude(pk__in=pks)
if stale.count() > 0:
logger.info("Deleting stale records (%s) for document %s" % (stale.count(), document))
stale.delete()
all_pks |= pks
if full:
stale = model.objects.exclude(pk__in=all_pks)
if stale.count() > 0:
logger.info("Deleting stale records (%s)" % stale.count())
stale.delete()
def merge_amount_dicts(a, b, merger):
return {
key: merger(a.get(key, 0), b.get(key, 0))
for key in (
set(six.iterkeys(a))
| set(six.iterkeys(b))
)
}
def test_dict_translation(self):
mvd = MultiValueDict({"devs": ["Bob", "Joe"], "pm": ["Rory"]})
d = mvd.dict()
self.assertEqual(sorted(six.iterkeys(d)), sorted(six.iterkeys(mvd)))
for key in six.iterkeys(mvd):
self.assertEqual(d[key], mvd[key])
self.assertEqual({}, MultiValueDict().dict())
def test_get_all_fields(self):
s = GroupSerializer()
all_keys1 = six.iterkeys(s.get_all_fields())
f2 = s.fields
all_keys2 = six.iterkeys(s.get_all_fields())
expected = ['id', 'name']
self.assertEqual(list(six.iterkeys(f2)), expected)
self.assertEqual(list(all_keys1), list(all_keys2))
def get_etag(self, request, obj, *args, **kwargs):
if obj.is_profile_visible(request.user):
return self.generate_etag(obj, six.iterkeys(self.fields), request)
else:
return self.generate_etag(obj, [
field
for field in six.iterkeys(self.fields)
if field not in self.hidden_fields
], request)
def test_get_fields_with_exclude_fields(self):
exclude = ['id']
expected = set(
six.iterkeys(UserSerializer().get_fields())
) - set(exclude)
serializer = UserSerializer(
exclude_fields=exclude,
)
self.assertEqual(set(six.iterkeys(serializer.fields)), expected)
def test_get_fields_with_include_fields(self):
include = ['permissions']
expected = set(
six.iterkeys(UserSerializer().get_fields())
) | set(include)
serializer = UserSerializer(
include_fields=include
)
self.assertEqual(set(six.iterkeys(serializer.fields)), expected)
def test_delete_and_insert(self):
"""
Deleting an item, then inserting the same key again will place it
at the end.
"""
del self.d2[7]
self.assertEqual(list(six.iterkeys(self.d2)), [1, 9, 0])
self.d2[7] = 'lucky number 7'
self.assertEqual(list(six.iterkeys(self.d2)), [1, 9, 0, 7])
def test_dict_translation(self):
mvd = MultiValueDict({
'devs': ['Bob', 'Joe'],
'pm': ['Rory'],
})
d = mvd.dict()
self.assertEqual(sorted(six.iterkeys(d)), sorted(six.iterkeys(mvd)))
for key in six.iterkeys(mvd):
self.assertEqual(d[key], mvd[key])
self.assertEqual({}, MultiValueDict().dict())
def _get_queryset(self, request, is_list=False, *args, **kwargs):
"""Returns an optimized queryset.
This calls out to the resource's get_queryset(), and then performs
some optimizations to better fetch related objects, reducing future
lookups in this request.
"""
queryset = self.get_queryset(request, is_list=is_list, *args, **kwargs)
if not hasattr(self, '_select_related_fields'):
self._select_related_fields = []
for field in six.iterkeys(self.fields):
if hasattr(self, 'serialize_%s_field' % field):
continue
field_type = getattr(self.model, field, None)
if (field_type and
isinstance(field_type,
ReverseSingleRelatedObjectDescriptor)):
self._select_related_fields.append(field)
if self._select_related_fields:
queryset = \
queryset.select_related(*self._select_related_fields)
if is_list:
if not hasattr(self, '_prefetch_related_fields'):
self._prefetch_related_fields = []
for field in six.iterkeys(self.fields):
if hasattr(self, 'serialize_%s_field' % field):
continue
field_type = getattr(self.model, field, None)
if (field_type and
isinstance(field_type,
(ReverseManyRelatedObjectsDescriptor,
ManyRelatedObjectsDescriptor))):
self._prefetch_related_fields.append(field)
if self._prefetch_related_fields:
queryset = \
queryset.prefetch_related(*self._prefetch_related_fields)
return queryset
def get_models(self, app_mod=None,
include_auto_created=False, include_deferred=False,
only_installed=True, include_swapped=False):
"""
Given a module containing models, returns a list of the models.
Otherwise returns a list of all installed models.
By default, auto-created models (i.e., m2m models without an
explicit intermediate table) are not included. However, if you
specify include_auto_created=True, they will be.
By default, models created to satisfy deferred attribute
queries are *not* included in the list of models. However, if
you specify include_deferred, they will be.
By default, models that aren't part of installed apps will *not*
be included in the list of models. However, if you specify
only_installed=False, they will be. If you're using a non-default
AppCache, this argument does nothing - all models will be included.
By default, models that have been swapped out will *not* be
included in the list of models. However, if you specify
include_swapped, they will be.
"""
if not self.loads_installed:
only_installed = False
cache_key = (app_mod, include_auto_created, include_deferred, only_installed, include_swapped)
model_list = None
try:
model_list = self._get_models_cache[cache_key]
if self.available_apps is not None and only_installed:
model_list = [m for m in model_list if m._meta.app_label in self.available_apps]
return model_list
except KeyError:
pass
self._populate()
if app_mod:
if app_mod in self.app_store:
app_list = [self.app_models.get(self._label_for(app_mod), ModelDict())]
else:
app_list = []
else:
if only_installed:
app_list = [self.app_models.get(app_label, ModelDict())
for app_label in six.iterkeys(self.app_labels)]
else:
app_list = six.itervalues(self.app_models)
model_list = []
for app in app_list:
model_list.extend(
model for model in app.values()
if ((not model._deferred or include_deferred) and
(not model._meta.auto_created or include_auto_created) and
(not model._meta.swapped or include_swapped))
)
self._get_models_cache[cache_key] = model_list
if self.available_apps is not None and only_installed:
model_list = [m for m in model_list if m._meta.app_label in self.available_apps]
return model_list
def get_indexes(self):
indexes = {
name: self.get_index(name)
for name in six.iterkeys(self._registry)
}
return indexes
def test_multiple_keys(self):
"""Test QueryDict with two key/value pairs with same keys."""
q = QueryDict(str('vote=yes&vote=no'))
self.assertEqual(q['vote'], 'no')
self.assertRaises(AttributeError, q.__setitem__, 'something', 'bar')
self.assertEqual(q.get('vote', 'default'), 'no')
self.assertEqual(q.get('foo', 'default'), 'default')
self.assertEqual(q.getlist('vote'), ['yes', 'no'])
self.assertEqual(q.getlist('foo'), [])
self.assertRaises(AttributeError, q.setlist, 'foo', ['bar', 'baz'])
self.assertRaises(AttributeError, q.setlist, 'foo', ['bar', 'baz'])
self.assertRaises(AttributeError, q.appendlist, 'foo', ['bar'])
if not six.PY3:
self.assertEqual(q.has_key('vote'), True)
self.assertEqual('vote' in q, True)
if not six.PY3:
self.assertEqual(q.has_key('foo'), False)
self.assertEqual('foo' in q, False)
self.assertEqual(list(six.iteritems(q)), [('vote', 'no')])
self.assertEqual(list(six.iterlists(q)), [('vote', ['yes', 'no'])])
self.assertEqual(list(six.iterkeys(q)), ['vote'])
self.assertEqual(list(six.itervalues(q)), ['no'])
self.assertEqual(len(q), 1)
self.assertRaises(AttributeError, q.update, {'foo': 'bar'})
self.assertRaises(AttributeError, q.pop, 'foo')
self.assertRaises(AttributeError, q.popitem)
self.assertRaises(AttributeError, q.clear)
self.assertRaises(AttributeError, q.setdefault, 'foo', 'bar')
self.assertRaises(AttributeError, q.__delitem__, 'vote')
def __init__(self, connection):
super(DatabaseOperations, self).__init__(connection)
# Determine the version of the SpatiaLite library.
try:
vtup = self.spatialite_version_tuple()
version = vtup[1:]
if version < (2, 3, 0):
raise ImproperlyConfigured('GeoDjango only supports SpatiaLite versions '
'2.3.0 and above')
self.spatial_version = version
except ImproperlyConfigured:
raise
except Exception as msg:
raise ImproperlyConfigured('Cannot determine the SpatiaLite version for the "%s" '
'database (error was "%s"). Was the SpatiaLite initialization '
'SQL loaded on this database?' %
(self.connection.settings_dict['NAME'], msg))
# Creating the GIS terms dictionary.
gis_terms = ['isnull']
gis_terms += list(six.iterkeys(self.geometry_functions))
self.gis_terms = dict([(term, None) for term in gis_terms])
if version >= (2, 4, 0):
# Spatialite 2.4.0-RC4 added AsGML and AsKML, however both
# RC2 (shipped in popular Debian/Ubuntu packages) and RC4
# report version as '2.4.0', so we fall back to feature detection
try:
self._get_spatialite_func("AsGML(GeomFromText('POINT(1 1)'))")
self.gml = 'AsGML'
self.kml = 'AsKML'
except DatabaseError:
# we are using < 2.4.0-RC4
pass
def resolve_columns(self, row, fields=()):
"""
This routine is necessary so that distances and geometries returned
from extra selection SQL get resolved appropriately into Python
objects.
"""
values = []
aliases = list(six.iterkeys(self.query.extra_select))
# Have to set a starting row number offset that is used for
# determining the correct starting row index -- needed for
# doing pagination with Oracle.
rn_offset = 0
if self.connection.ops.oracle:
if self.query.high_mark is not None or self.query.low_mark: rn_offset = 1
index_start = rn_offset + len(aliases)
# Converting any extra selection values (e.g., geometries and
# distance objects added by GeoQuerySet methods).
values = [self.query.convert_values(v,
self.query.extra_select_fields.get(a, None),
self.connection)
for v, a in zip(row[rn_offset:index_start], aliases)]
if self.connection.ops.oracle or getattr(self.query, 'geo_values', False):
# We resolve the rest of the columns if we're on Oracle or if
# the `geo_values` attribute is defined.
for value, field in map(None, row[index_start:], fields):
values.append(self.query.convert_values(value, field, self.connection))
else:
values.extend(row[index_start:])
return tuple(values)
def keys_with_prefix(self, prefix, version=None):
if self.reverse_key_func is None:
raise ValueError(
"To use the _with_prefix commands with a custom KEY_FUNCTION, "
"you need to specify a custom REVERSE_KEY_FUNCTION too."
)
if version is None:
version = self.version
db = router.db_for_read(self.cache_model_class)
table = connections[db].ops.quote_name(self._table)
prefix = self.make_key(prefix + '%', version=version)
with connections[db].cursor() as cursor:
cursor.execute(
"""SELECT cache_key FROM {table}
WHERE cache_key LIKE %s AND
expires >= %s""".format(table=table),
(prefix, self._now())
)
rows = cursor.fetchall()
full_keys = {row[0] for row in rows}
keys = {}
for full_key in full_keys:
key, key_prefix, key_version = self.reverse_key_func(full_key)
if key_version == version:
keys[key] = key_version
return set(six.iterkeys(keys))
def similar_objects(self):
lookup_kwargs = self._lookup_kwargs()
lookup_keys = sorted(lookup_kwargs)
qs = self.through.objects.values(*six.iterkeys(lookup_kwargs))
qs = qs.annotate(n=models.Count("pk"))
qs = qs.exclude(**lookup_kwargs)
qs = qs.filter(tag__in=self.all())
qs = qs.order_by("-n")
# TODO: This all feels like a bit of a hack.
items = {}
if len(lookup_keys) == 1:
# Can we do this without a second query by using a select_related()
# somehow?
f = _get_field(self.through, lookup_keys[0])
objs = f.rel.to._default_manager.filter(**{"%s__in" % f.rel.field_name: [r["content_object"] for r in qs]})
for obj in objs:
items[(getattr(obj, f.rel.field_name),)] = obj
else:
preload = {}
for result in qs:
preload.setdefault(result["content_type"], set())
preload[result["content_type"]].add(result["object_id"])
for ct, obj_ids in preload.items():
ct = ContentType.objects.get_for_id(ct)
for obj in ct.model_class()._default_manager.filter(pk__in=obj_ids):
items[(ct.pk, obj.pk)] = obj
results = []
for result in qs:
obj = items[tuple(result[k] for k in lookup_keys)]
obj.similar_tags = result["n"]
results.append(obj)
return results
def test_multiple_keys(self):
"""Test QueryDict with two key/value pairs with same keys."""
q = QueryDict(str("vote=yes&vote=no"))
self.assertEqual(q["vote"], "no")
self.assertRaises(AttributeError, q.__setitem__, "something", "bar")
self.assertEqual(q.get("vote", "default"), "no")
self.assertEqual(q.get("foo", "default"), "default")
self.assertEqual(q.getlist("vote"), ["yes", "no"])
self.assertEqual(q.getlist("foo"), [])
self.assertRaises(AttributeError, q.setlist, "foo", ["bar", "baz"])
self.assertRaises(AttributeError, q.setlist, "foo", ["bar", "baz"])
self.assertRaises(AttributeError, q.appendlist, "foo", ["bar"])
if six.PY2:
self.assertEqual(q.has_key("vote"), True)
self.assertEqual("vote" in q, True)
if six.PY2:
self.assertEqual(q.has_key("foo"), False)
self.assertEqual("foo" in q, False)
self.assertEqual(list(six.iteritems(q)), [("vote", "no")])
self.assertEqual(list(six.iterlists(q)), [("vote", ["yes", "no"])])
self.assertEqual(list(six.iterkeys(q)), ["vote"])
self.assertEqual(list(six.itervalues(q)), ["no"])
self.assertEqual(len(q), 1)
self.assertRaises(AttributeError, q.update, {"foo": "bar"})
self.assertRaises(AttributeError, q.pop, "foo")
self.assertRaises(AttributeError, q.popitem)
self.assertRaises(AttributeError, q.clear)
self.assertRaises(AttributeError, q.setdefault, "foo", "bar")
self.assertRaises(AttributeError, q.__delitem__, "vote")
def serialize_change_entry(self, changedesc):
"""Serialize a change entry for public consumption.
This will output a version of the change entry for use in the API.
It can be the same content stored in the
:py:class:`~reviewboard.changedescs.models.ChangeDescription`, but
does not need to be.
Args:
changedesc (reviewboard.changedescs.models.ChangeDescription):
The change description whose field is to be serialized.
Returns:
list:
An appropriate serialization for the field.
"""
data = changedesc.fields_changed[self.field_id]
return [
{
'old': data[six.text_type(obj.pk)]['old'][0],
'new': data[six.text_type(obj.pk)]['new'][0],
self.caption_object_field: obj,
}
for obj in self.model.objects.filter(pk__in=six.iterkeys(data))
]
def collect(self, request=None, files=[]):
if self.request and self.request is request:
return
self.request = request
found_files = OrderedDict()
for finder in finders.get_finders():
# Ignore our finder to avoid looping
if isinstance(finder, PipelineFinder):
continue
for path, storage in finder.list(['CVS', '.*', '*~']):
# Prefix the relative path if the source storage contains it
if getattr(storage, 'prefix', None):
prefixed_path = os.path.join(storage.prefix, path)
else:
prefixed_path = path
if (prefixed_path not in found_files and
(not files or prefixed_path in files)):
found_files[prefixed_path] = (storage, path)
self.copy_file(path, prefixed_path, storage)
if files and len(files) == len(found_files):
break
return six.iterkeys(found_files)
def __new__(cls, name, bases, attrs):
fields = [(field_name, attrs.pop(field_name))
for field_name, obj in list(six.iteritems(attrs))
if isinstance(obj, SearchField)]
fields.sort(key=lambda x: x[1].creation_counter)
for base in bases[::-1]:
if hasattr(base, 'declared_fields'):
fields = list(six.iteritems(base.declared_fields)) + fields
field_dict = dict(fields)
for k in list(six.iterkeys(field_dict)):
if k in attrs and attrs[k] is None:
del field_dict[k]
attrs['declared_fields'] = field_dict
new_class = super(DeclarativeSearchFieldMetaclass, cls).__new__(cls, name, bases, attrs)
OptionsClass = new_class._options_class
if OptionsClass:
options_list = [c.Meta for c in new_class.mro() if hasattr(c, 'Meta')]
new_class._meta = OptionsClass(options_list)
return new_class
def formfield(self, form_class=forms.CharField, **kwargs):
"""
Returns a django.forms.Field instance for this database Field.
"""
defaults = {'required': not self.blank,
'label': capfirst(self.verbose_name),
'help_text': self.help_text}
if self.has_default():
if callable(self.default):
defaults['initial'] = self.default
defaults['show_hidden_initial'] = True
else:
defaults['initial'] = self.get_default()
if self.choices:
# Fields with choices get special treatment.
include_blank = (self.blank or
not (self.has_default() or 'initial' in kwargs))
defaults['choices'] = self.get_choices(include_blank=include_blank)
defaults['coerce'] = self.to_python
if self.null:
defaults['empty_value'] = None
form_class = forms.TypedChoiceField
# Many of the subclass-specific formfield arguments (min_value,
# max_value) don't apply for choice fields, so be sure to only pass
# the values that TypedChoiceField will understand.
for k in list(six.iterkeys(kwargs)):
if k not in ('coerce', 'empty_value', 'choices', 'required',
'widget', 'label', 'initial', 'help_text',
'error_messages', 'show_hidden_initial'):
del kwargs[k]
defaults.update(kwargs)
return form_class(**defaults)
def test_mergedict_merges_multivaluedict(self):
""" MergeDict can merge MultiValueDicts """
multi1 = MultiValueDict({'key1': ['value1'],
'key2': ['value2', 'value3']})
multi2 = MultiValueDict({'key2': ['value4'],
'key4': ['value5', 'value6']})
mm = MergeDict(multi1, multi2)
# Although 'key2' appears in both dictionaries,
# only the first value is used.
self.assertEqual(mm.getlist('key2'), ['value2', 'value3'])
self.assertEqual(mm.getlist('key4'), ['value5', 'value6'])
self.assertEqual(mm.getlist('undefined'), [])
self.assertEqual(sorted(six.iterkeys(mm)), ['key1', 'key2', 'key4'])
self.assertEqual(len(list(six.itervalues(mm))), 3)
self.assertTrue('value1' in six.itervalues(mm))
self.assertEqual(sorted(six.iteritems(mm), key=lambda k: k[0]),
[('key1', 'value1'), ('key2', 'value3'),
('key4', 'value6')])
self.assertEqual([(k,mm.getlist(k)) for k in sorted(mm)],
[('key1', ['value1']),
('key2', ['value2', 'value3']),
('key4', ['value5', 'value6'])])
def _to_xml(self, xml, data, tag_name=None):
if tag_name:
xml.startElement(tag_name, {})
if hasattr(data, 'write_xml'):
# Support for the `IWriteXML` protocol.
data.write_xml(xml)
elif data is True:
xml.characters('true')
elif data is False:
xml.characters('false')
elif isinstance(data, (list, tuple)):
for item in data:
self._to_xml(xml, item, tag_name=(getattr(data, "xml_tag", None) or self.item_tag_name))
elif isinstance(data, dict):
key_order = getattr(data, "key_order", ())
for key in sorted(six.iterkeys(data), key=order_by_sort_order(key_order)):
self._to_xml(xml, data[key], key)
elif data is None: # Don't output any value
pass
else:
xml.characters(smart_text(data))
if tag_name:
xml.endElement(tag_name)
def templatetag(parser, token):
"""
Outputs one of the bits used to compose template tags.
Since the template system has no concept of "escaping", to display one of
the bits used in template tags, you must use the ``{% templatetag %}`` tag.
The argument tells which template bit to output:
================== =======
Argument Outputs
================== =======
``openblock`` ``{%``
``closeblock`` ``%}``
``openvariable`` ``{{``
``closevariable`` ``}}``
``openbrace`` ``{``
``closebrace`` ``}``
``opencomment`` ``{#``
``closecomment`` ``#}``
================== =======
"""
bits = token.contents.split()
if len(bits) != 2:
raise TemplateSyntaxError("'templatetag' statement takes one argument")
tag = bits[1]
if tag not in TemplateTagNode.mapping:
raise TemplateSyntaxError("Invalid templatetag argument: '%s'."
" Must be one of: %s" %
(tag, list(six.iterkeys(TemplateTagNode.mapping))))
return TemplateTagNode(tag)
def test_mergedict_merges_multivaluedict(self):
""" MergeDict can merge MultiValueDicts """
multi1 = MultiValueDict({"key1": ["value1"], "key2": ["value2", "value3"]})
multi2 = MultiValueDict({"key2": ["value4"], "key4": ["value5", "value6"]})
mm = MergeDict(multi1, multi2)
# Although 'key2' appears in both dictionaries,
# only the first value is used.
self.assertEqual(mm.getlist("key2"), ["value2", "value3"])
self.assertEqual(mm.getlist("key4"), ["value5", "value6"])
self.assertEqual(mm.getlist("undefined"), [])
self.assertEqual(sorted(six.iterkeys(mm)), ["key1", "key2", "key4"])
self.assertEqual(len(list(six.itervalues(mm))), 3)
self.assertIn("value1", six.itervalues(mm))
self.assertEqual(
sorted(six.iteritems(mm), key=lambda k: k[0]), [("key1", "value1"), ("key2", "value3"), ("key4", "value6")]
)
self.assertEqual(
[(k, mm.getlist(k)) for k in sorted(mm)],
[("key1", ["value1"]), ("key2", ["value2", "value3"]), ("key4", ["value5", "value6"])],
)
def _perform_delete_app_test(self, sql_name, database=None):
# Simulate deletion of the app.
self.set_base_model(
self.default_base_model,
extra_models=self.default_extra_models,
db_name=database)
end_sig = self.start_sig.clone()
end_sig.remove_app_sig('tests')
d = Diff(self.start_sig, end_sig)
self.assertEqual(sorted(six.iterkeys(d.deleted)), ['tests'])
self.assertEqual(d.deleted['tests'],
['TestModel', 'AppDeleteAnchor1', 'AppDeleteAnchor2',
'CustomTestModel'])
mutation = DeleteApplication()
self.perform_simulations([mutation], end_sig, ignore_apps=True,
db_name=database)
test_database_state = self.database_state.clone()
test_sig = self.start_sig.clone()
app_mutator = AppMutator(app_label='tests',
project_sig=test_sig,
database_state=test_database_state,
database=database)
app_mutator.run_mutation(mutation)
sql = app_mutator.to_sql()
self.assertEqual('\n'.join(sql),
self.get_sql_mapping(sql_name, database))
def list(self):
return six.iterkeys(self._users)
def list(self, service, user=None):
if user is None:
return six.iterkeys(self._groups[self._service(service)])
else:
groups = self._groups[self._service(service)]
return [k for k, v in six.iteritems(groups) if v.is_member(user)]
def get_models(self):
"""
Get all models in the registry
"""
return set(iterkeys(self._models))
setting_changed.connect(settings_changed_handler)
class CachedSettingsMixin(object):
def __init__(self, *args, **kwargs):
self.update_patterns()
super(CachedSettingsMixin, self).__init__(*args, **kwargs)
def update_patterns(self):
if not self.js_assets_repl_enabled:
return
esc_tag = re.escape(self.js_assets_repl_tag)
self.patterns += (("*.js", ((
r"""(/\*!\s*%s(?:\((.*?)\))?\s*\*/\s*['"](.*?)['"]\s*/\*!\s*end%s\s*\*/(\n)?)"""
% (esc_tag, esc_tag), """'%s'"""), )), )
class SettingProxy(object):
def __init__(self, key):
self.key = key
def __call__(self, instance):
return get_cached_setting_key(self.key)
# Dynamically create properties, whose names are lower-cased keys of
# settings_defaults
for key in iterkeys(settings_defaults):
setattr(CachedSettingsMixin, key.lower(), property(SettingProxy(key)))
def serialize_object(self, obj, *args, **kwargs):
"""Serializes the object, transforming text fields.
This is a specialization of serialize_object that transforms any
text fields that support text types. It also handles attaching
the raw text to the payload, on request.
"""
data = super(MarkdownFieldsMixin,
self).serialize_object(obj, *args, **kwargs)
request = kwargs.get('request')
if not request:
force_text_type = None
elif request.method == 'GET':
force_text_type = request.GET.get('force-text-type')
else:
force_text_type = request.POST.get('force_text_type')
if force_text_type not in self.TEXT_TYPES:
force_text_type = None
extra_text_type_fields = dict(
(extra_text_type, {})
for extra_text_type in self._get_extra_text_types(obj, **kwargs))
for field, field_info in six.iteritems(self.fields):
if not field_info.get('supports_text_types'):
continue
get_func = getattr(self, 'get_is_%s_rich_text' % field, None)
if six.callable(get_func):
getter = lambda obj, *args: get_func(obj)
else:
getter = lambda obj, data, rich_text_field, text_type_field: \
getattr(obj, rich_text_field, None)
self._serialize_text_info(obj, data, extra_text_type_fields, field,
force_text_type, getter)
if 'extra_data' in data:
extra_data = data['extra_data']
all_text_types_extra_data = {}
# Work on a copy of extra_data, in case we change it.
for field, value in six.iteritems(obj.extra_data.copy()):
if not self.get_extra_data_field_supports_markdown(obj, field):
continue
# If all_text_types_extra_data is empty that implies we have
# encountered the first field in extra_data which supports
# markdown. In this case we must initialize the dictionary
# with the extra text types that should be included in the
# payload.
if not all_text_types_extra_data:
all_text_types_extra_data = dict(
(k, {}) for k in six.iterkeys(extra_text_type_fields))
# Note that we assume all custom fields are in Markdown by
# default. This is to preserve compatibility with older
# fields. New fields will always have the text_type flag
# set to the proper value.
self._serialize_text_info(obj, extra_data,
all_text_types_extra_data, field,
force_text_type,
self._extra_data_rich_text_getter)
for key, values in six.iteritems(all_text_types_extra_data):
extra_text_type_fields[key]['extra_data'] = values
for key, values in six.iteritems(extra_text_type_fields):
data[key + '_text_fields'] = values
return data
def get_models(self,
app_mod=None,
include_auto_created=False,
include_deferred=False,
only_installed=True,
include_swapped=False):
"""
Given a module containing models, returns a list of the models.
Otherwise returns a list of all installed models.
By default, auto-created models (i.e., m2m models without an
explicit intermediate table) are not included. However, if you
specify include_auto_created=True, they will be.
By default, models created to satisfy deferred attribute
queries are *not* included in the list of models. However, if
you specify include_deferred, they will be.
By default, models that aren't part of installed apps will *not*
be included in the list of models. However, if you specify
only_installed=False, they will be.
By default, models that have been swapped out will *not* be
included in the list of models. However, if you specify
include_swapped, they will be.
"""
cache_key = (app_mod, include_auto_created, include_deferred,
only_installed, include_swapped)
model_list = None
try:
model_list = self._get_models_cache[cache_key]
if self.available_apps is not None and only_installed:
model_list = [
m for m in model_list
if m._meta.app_label in self.available_apps
]
return model_list
except KeyError:
pass
self._populate()
if app_mod:
if app_mod in self.app_store:
app_list = [
self.app_models.get(self._label_for(app_mod), SortedDict())
]
else:
app_list = []
else:
if only_installed:
app_list = [
self.app_models.get(app_label, SortedDict())
for app_label in six.iterkeys(self.app_labels)
]
else:
app_list = six.itervalues(self.app_models)
model_list = []
for app in app_list:
model_list.extend(
model for model in app.values()
if ((not model._deferred or include_deferred) and (
not model._meta.auto_created or include_auto_created) and (
not model._meta.swapped or include_swapped)))
self._get_models_cache[cache_key] = model_list
if self.available_apps is not None and only_installed:
model_list = [
m for m in model_list
if m._meta.app_label in self.available_apps
]
return model_list
def changed_fields(self):
return (set(name for name in six.iterkeys(self._changed_fields))
| set(name
for name, value in six.iteritems(self._mutable_fields)
if getattr(self, name, DoesNotExist) != value))
def _get_real_instances(self, base_result_objects):
"""
Polymorphic object loader
Does the same as:
return [ o.get_real_instance() for o in base_result_objects ]
but more efficiently.
The list base_result_objects contains the objects from the executed
base class query. The class of all of them is self.model (our base model).
Some, many or all of these objects were not created and stored as
class self.model, but as a class derived from self.model. We want to re-fetch
these objects from the db as their original class so we can return them
just as they were created/saved.
We identify these objects by looking at o.polymorphic_ctype, which specifies
the real class of these objects (the class at the time they were saved).
First, we sort the result objects in base_result_objects for their
subclass (from o.polymorphic_ctype), and then we execute one db query per
subclass of objects. Here, we handle any annotations from annotate().
Finally we re-sort the resulting objects into the correct order and
return them as a list.
"""
ordered_id_list = [] # list of ids of result-objects in correct order
results = {} # polymorphic dict of result-objects, keyed with their id (no order)
# dict contains one entry per unique model type occurring in result,
# in the format idlist_per_model[modelclass]=[list-of-object-ids]
idlist_per_model = defaultdict(list)
# django's automatic ".pk" field does not always work correctly for
# custom fields in derived objects (unclear yet who to put the blame on).
# We get different type(o.pk) in this case.
# We work around this by using the real name of the field directly
# for accessing the primary key of the the derived objects.
# We might assume that self.model._meta.pk.name gives us the name of the primary key field,
# but it doesn't. Therefore we use polymorphic_primary_key_name, which we set up in base.py.
pk_name = self.model.polymorphic_primary_key_name
# - sort base_result_object ids into idlist_per_model lists, depending on their real class;
# - also record the correct result order in "ordered_id_list"
# - store objects that already have the correct class into "results"
base_result_objects_by_id = {}
content_type_manager = ContentType.objects.db_manager(self.db)
self_model_class_id = content_type_manager.get_for_model(self.model, for_concrete_model=False).pk
self_concrete_model_class_id = content_type_manager.get_for_model(self.model, for_concrete_model=True).pk
for base_object in base_result_objects:
ordered_id_list.append(base_object.pk)
# check if id of the result object occurres more than once - this can happen e.g. with base_objects.extra(tables=...)
if base_object.pk not in base_result_objects_by_id:
base_result_objects_by_id[base_object.pk] = base_object
if base_object.polymorphic_ctype_id == self_model_class_id:
# Real class is exactly the same as base class, go straight to results
results[base_object.pk] = base_object
else:
real_concrete_class = base_object.get_real_instance_class()
real_concrete_class_id = base_object.get_real_concrete_instance_class_id()
if real_concrete_class_id is None:
# Dealing with a stale content type
continue
elif real_concrete_class_id == self_concrete_model_class_id:
# Real and base classes share the same concrete ancestor,
# upcast it and put it in the results
results[base_object.pk] = transmogrify(real_concrete_class, base_object)
else:
real_concrete_class = content_type_manager.get_for_id(real_concrete_class_id).model_class()
idlist_per_model[real_concrete_class].append(getattr(base_object, pk_name))
# For each model in "idlist_per_model" request its objects (the real model)
# from the db and store them in results[].
# Then we copy the annotate fields from the base objects to the real objects.
# Then we copy the extra() select fields from the base objects to the real objects.
# TODO: defer(), only(): support for these would be around here
for real_concrete_class, idlist in idlist_per_model.items():
real_objects = real_concrete_class.base_objects.db_manager(self.db).filter(**{
('%s__in' % pk_name): idlist,
})
real_objects.query.select_related = self.query.select_related # copy select related configuration to new qs
# Copy deferred fields configuration to the new queryset
deferred_loading_fields = []
existing_fields = self.polymorphic_deferred_loading[0]
for field in existing_fields:
try:
translated_field_name = translate_polymorphic_field_path(
real_concrete_class, field)
except AssertionError:
if '___' in field:
# The originally passed argument to .defer() or .only()
# was in the form Model2B___field2, where Model2B is
# now a superclass of real_concrete_class. Thus it's
# sufficient to just use the field name.
translated_field_name = field.rpartition('___')[-1]
else:
raise
deferred_loading_fields.append(translated_field_name)
real_objects.query.deferred_loading = (set(deferred_loading_fields), self.query.deferred_loading[1])
for real_object in real_objects:
o_pk = getattr(real_object, pk_name)
real_class = real_object.get_real_instance_class()
# If the real class is a proxy, upcast it
if real_class != real_concrete_class:
real_object = transmogrify(real_class, real_object)
if _query_annotations(self.query):
for anno_field_name in six.iterkeys(_query_annotations(self.query)):
attr = getattr(base_result_objects_by_id[o_pk], anno_field_name)
setattr(real_object, anno_field_name, attr)
if self.query.extra_select:
for select_field_name in six.iterkeys(self.query.extra_select):
attr = getattr(base_result_objects_by_id[o_pk], select_field_name)
setattr(real_object, select_field_name, attr)
results[o_pk] = real_object
# re-create correct order and return result list
resultlist = [results[ordered_id] for ordered_id in ordered_id_list if ordered_id in results]
# set polymorphic_annotate_names in all objects (currently just used for debugging/printing)
if _query_annotations(self.query):
annotate_names = list(six.iterkeys(_query_annotations(self.query))) # get annotate field list
for real_object in resultlist:
real_object.polymorphic_annotate_names = annotate_names
# set polymorphic_extra_select_names in all objects (currently just used for debugging/printing)
if self.query.extra_select:
extra_select_names = list(six.iterkeys(self.query.extra_select)) # get extra select field list
for real_object in resultlist:
real_object.polymorphic_extra_select_names = extra_select_names
return resultlist
def search(self, query, search_fields=None, order_results=True,
decorate_results=True):
"""
Build a queryset matching words in the given search query, treating
quoted terms as exact phrases and taking into account + and - symbols
as modifiers controlling which terms to require and exclude.
"""
assert self.query.can_filter(), \
'Cannot filter a query once a slice has been taken.'
helper = import_string(self._search_helper)
queryset = self._clone()
queryset._search_ordered = order_results
queryset._search_decorated = order_results or decorate_results
#### DETERMINE FIELDS TO SEARCH ###
# Use search_fields argument if given, otherwise use queryset._search_fields
# property (which is initially configured by the manager class).
if search_fields:
queryset._search_fields = search_fields_to_dict(search_fields)
if not queryset._search_fields:
return queryset.none()
#### BUILD LIST OF TERMS TO SEARCH FOR ###
# Remove extra spaces, put modifiers inside quoted terms.
terms = ' '.join(query.split()).replace('+"', '"+') \
.replace('-"', '"-') \
.split('"')
# Strip punctuation other than modifiers from terms and create terms
# list, first from quoted terms and then remaining words.
terms = [
(t[0] if t.startswith(('-', '+')) else '') + t.strip(punctuation)
for t
in terms[1::2] + ''.join(terms[::2]).split()
]
# Remove stop words from terms that aren't quoted or use modifiers,
# since words with these are an explicit part of the search query.
# If doing so ends up with an empty term list, then keep the stop
# words.
terms_no_stopwords = [
t
for t in terms
if t.startswith(('-', '+', '"'))
or t.lower() not in registry['core:STOP_WORDS']
]
positive_terms = [
(t if not t.startswith('+') else t[1:]).lower()
for t in terms_no_stopwords
if not t.startswith('-')
]
if not positive_terms:
positive_terms = [
(t if not t.startswith('+') else t[1:]).lower()
for t in terms
if not t.startswith('-')
]
else:
terms = terms_no_stopwords
# Avoid too short or too long queries.
query_len = len(''.join(terms))
if (query_len < settings.SEARCH_MIN_QUERY_LEN
or query_len > settings.SEARCH_MAX_QUERY_LEN):
return queryset.none()
# Remove too short words.
positive_terms = [
t
for t in positive_terms
if len(t) >= settings.SEARCH_MIN_WORD_LEN
]
terms = [
t
for t in terms
if len(t.strip('-+')) >= settings.SEARCH_MIN_WORD_LEN
]
# Append positive terms (those without the negative modifier) to the
# internal list for sorting when results are iterated.
if not positive_terms:
return queryset.none()
else:
queryset._search_terms.update(positive_terms)
### BUILD QUERYSET FILTER ###
engine = connections[queryset.db].vendor
# Filter the queryset combining each set of terms.
field_lookups = [
helper.prepare_field_lookup(f)
for f
in six.iterkeys(queryset._search_fields)
]
excluded = []
required = []
optional = []
for t in terms:
if t.startswith('-'):
term = helper.prepare_term(t[1:], engine)
excluded.append(reduce(
iand,
[~Q(**{lookup: term}) for lookup in field_lookups]
))
elif t.startswith('+'):
term = helper.prepare_term(t[1:], engine)
required.append(reduce(
ior,
[Q(**{lookup: term}) for lookup in field_lookups]
))
else:
term = helper.prepare_term(t, engine)
optional.append(reduce(
ior,
[Q(**{lookup: term}) for lookup in field_lookups]
))
queryset.query.add_distinct_fields()
queryset.query.clear_ordering(force_empty=True)
if excluded:
queryset = queryset.filter(reduce(iand, excluded))
if required:
queryset = queryset.filter(reduce(iand, required))
elif optional:
# Optional terms aren't relevant to the filter if there are terms
# that are explicitly required.
queryset = queryset.filter(reduce(ior, optional))
return queryset
class KeyTransform(Transform):
SPEC_MAP = {
date: 'DATE',
datetime: 'DATETIME',
float: 'DOUBLE',
int: 'INTEGER',
six.text_type: 'CHAR',
time: 'TIME',
dict: 'BINARY',
}
if six.PY2:
from __builtin__ import long # make source lintable on Python 3
SPEC_MAP[long] = 'INTEGER'
SPEC_MAP_NAMES = ', '.join(sorted(x.__name__ for x in
six.iterkeys(SPEC_MAP)))
TYPE_MAP = {
'BINARY': DynamicField,
'CHAR': TextField(),
'DATE': DateField(),
'DATETIME': DateTimeField(),
'DOUBLE': FloatField(),
'INTEGER': IntegerField(),
'TIME': TimeField(),
}
def __init__(self, key_name, data_type, *args, **kwargs):
subspec = kwargs.pop('subspec', None)
super(KeyTransform, self).__init__(*args, **kwargs)
self.key_name = key_name
self.data_type = data_type
try:
output_field = self.TYPE_MAP[data_type]
except KeyError: # pragma: no cover
raise ValueError("Invalid data_type '{}'".format(data_type))
if data_type == 'BINARY':
self.output_field = output_field(spec=subspec)
else:
self.output_field = output_field
def as_sql(self, compiler, connection):
lhs, params = compiler.compile(self.lhs)
return (
"COLUMN_GET({}, %s AS {})".format(lhs, self.data_type),
params + [self.key_name],
)
if django.VERSION[:3] <= (1, 8, 2): # pragma: no cover
# Backport of bugfix for transforms with arguments, taken from:
# https://code.djangoproject.com/ticket/24744
def copy(self):
return copy(self)
def relabeled_clone(self, relabels):
copy = self.copy()
copy.lhs = self.lhs.relabeled_clone(relabels)
return copy
def copy(self):
other = self.__class__()
for key in six.iterkeys(self):
other[key] = self[key]
return other
def test_get_fields_with_only_fields(self):
expected = ['id', 'last_name']
serializer = UserSerializer(only_fields=expected)
self.assertEqual(list(six.iterkeys(serializer.fields)), expected)
def test_copy(self):
orig = SortedDict(((1, "one"), (0, "zero"), (2, "two")))
copied = copy.copy(orig)
self.assertEqual(list(six.iterkeys(orig)), [1, 0, 2])
self.assertEqual(list(six.iterkeys(copied)), [1, 0, 2])
def reverted(self):
return dict(
six.moves.zip(six.itervalues(self.enum_dict),
six.iterkeys(self.enum_dict)))
def test_append_items(self):
""" New items go to the end. """
self.d1[0] = 'nil'
self.assertEqual(list(six.iterkeys(self.d1)), [7, 1, 9, 0])
def test_get_fields_with_include_all(self):
expected = six.iterkeys(UserSerializer().get_all_fields())
serializer = UserSerializer(
include_fields='*'
)
self.assertEqual(list(six.iterkeys(serializer.fields)), list(expected))
def test_overwrite(self):
self.d1[1] = 'not one'
self.assertEqual(self.d1[1], 'not one')
self.assertEqual(list(six.iterkeys(self.d1)),
list(six.iterkeys(self.d1.copy())))
def keys(self):
return six.iterkeys(self.enum_dict)
def test_basic_methods(self):
self.assertEqual(list(six.iterkeys(self.d1)), [7, 1, 9])
self.assertEqual(list(six.itervalues(self.d1)),
['seven', 'one', 'nine'])
self.assertEqual(list(six.iteritems(self.d1)),
[(7, 'seven'), (1, 'one'), (9, 'nine')])