def _get_model_from_sheet(self, sheet):
"""
Get the model name for the given sheet. Handles parsing out the comments
"""
# The model name is the section before the first space (if there is one)
model_name = sheet.name.split(' ')[0]
return python_serializer._get_model(model_name)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
money_fields = {}
fields = {}
Model = _get_model(obj["model"])
for (field_name, field_value) in six.iteritems(obj['fields']):
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(obj.object, field, value)
yield obj
except GeneratorExit:
raise
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
objects = json.loads(stream_or_string)
for obj in objects:
Model = _get_model(obj['model'])
if isinstance(obj['pk'], (tuple, list)):
try:
o = Model.objects.get_by_natural_key(*obj['pk'])
except Model.DoesNotExist:
obj['pk'] = None
else:
obj['pk'] = o.pk
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
money_fields = {}
fields = {}
Model = _get_model(obj["model"])
for (field_name, field_value) in obj['fields'].iteritems():
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Decimal(
field_value.split(" ")[0])
else:
fields[field_name] = field_value
obj['fields'] = fields
for obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(obj.object, field, value)
yield obj
except GeneratorExit:
raise
def Deserializer(object_list, **options):
"""Deserialize simple Python objects back into Model instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = python._get_model(d["model"])
data = {}
key = resolve_key(Model._meta.module_name, d["pk"])
if key.name():
data["key_name"] = key.name()
parent = None
if key.parent():
parent = FakeParent(key.parent())
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value,
options.get(
"encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model.properties()[field_name]
if isinstance(field, db.Reference):
# Resolve foreign key references.
data[field.name] = resolve_key(Model._meta.module_name,
field_value)
else:
# Handle converting strings to more specific formats.
if isinstance(field_value, basestring):
if isinstance(field, db.DateProperty):
field_value = datetime.datetime.strptime(
field_value, '%Y-%m-%d').date()
elif isinstance(field, db.TimeProperty):
field_value = parse_datetime_with_microseconds(
field_value, '%H:%M:%S').time()
elif isinstance(field, db.DateTimeProperty):
field_value = parse_datetime_with_microseconds(
field_value, '%Y-%m-%d %H:%M:%S')
# Handle pyyaml datetime.time deserialization - it returns a datetime
# instead of a time.
if (isinstance(field_value, datetime.datetime)
and isinstance(field, db.TimeProperty)):
field_value = field_value.time()
data[field.name] = field.validate(field_value)
# Create the new model instance with all it's data, but no parent.
object = Model(**data)
# Now add the parent into the hidden attribute, bypassing the type checks
# in the Model's __init__ routine.
object._parent = parent
# When the deserialized object is saved our replacement DeserializedObject
# class will set object._parent to force the real parent model to be loaded
# the first time it is referenced.
yield base.DeserializedObject(object, m2m_data)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, str):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
models.get_apps()
object_list = simplejson.load(stream)
if not isinstance(object_list, list):
object_list = [object_list]
for obj in object_list:
# Look up the model and starting build a dict of data for it.
if 'screen_name' in obj:
Model = _get_model('twitter_roa.user')
else:
Model = _get_model("twitter_roa.tweet")
data = {}
m2m_data = {}
# Handle each field
for (field_name, field_value) in obj.items():
if isinstance(field_value, str):
field_value = smart_unicode(field_value,
options.get(
"encoding", DEFAULT_CHARSET),
strings_only=True)
try:
field = Model._meta.get_field(field_name)
except models.FieldDoesNotExist:
continue
# Handle FK fields
if field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
data[field.attname] = field.rel.to._meta.\
get_field(field.rel.field_name).\
to_python(field_value['id'])
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
models.get_apps()
object_list = simplejson.load(stream)
if not isinstance(object_list, list):
object_list = [object_list]
for obj in object_list:
# Look up the model and starting build a dict of data for it.
if 'screen_name' in obj:
Model = _get_model('twitter_roa.user')
else:
Model = _get_model("twitter_roa.tweet")
data = {}
m2m_data = {}
# Handle each field
for (field_name, field_value) in obj.iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(
field_value,
options.get("encoding", DEFAULT_CHARSET),
strings_only=True)
try:
field = Model._meta.get_field(field_name)
except models.FieldDoesNotExist:
continue
# Handle FK fields
if field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
data[field.attname] = field.rel.to._meta.\
get_field(field.rel.field_name).\
to_python(field_value['id'])
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def prepare_object(value):
# Fix and prepare object before deserialization
_Model = _get_model(value["model"])
fields = get_fields_from_model(_Model)
for _field in list(value["fields"].keys()):
if _field not in fields:
del value["fields"][_field]
return value
def Deserializer(object_list, **options):
"""Deserialize simple Python objects back into Model instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = python._get_model(d["model"])
data = {}
key = resolve_key(Model._meta.module_name, d["pk"])
if key.name():
data["key_name"] = key.name()
parent = None
if key.parent():
parent = FakeParent(key.parent())
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(
field_value, options.get("encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model.properties()[field_name]
if isinstance(field, db.Reference):
# Resolve foreign key references.
data[field.name] = resolve_key(Model._meta.module_name, field_value)
else:
# Handle converting strings to more specific formats.
if isinstance(field_value, basestring):
if isinstance(field, db.DateProperty):
field_value = datetime.datetime.strptime(
field_value, '%Y-%m-%d').date()
elif isinstance(field, db.TimeProperty):
field_value = parse_datetime_with_microseconds(field_value,
'%H:%M:%S').time()
elif isinstance(field, db.DateTimeProperty):
field_value = parse_datetime_with_microseconds(field_value,
'%Y-%m-%d %H:%M:%S')
# Handle pyyaml datetime.time deserialization - it returns a datetime
# instead of a time.
if (isinstance(field_value, datetime.datetime) and
isinstance(field, db.TimeProperty)):
field_value = field_value.time()
data[field.name] = field.validate(field_value)
# Create the new model instance with all it's data, but no parent.
object = Model(**data)
# Now add the parent into the hidden attribute, bypassing the type checks
# in the Model's __init__ routine.
object._parent = parent
# When the deserialized object is saved our replacement DeserializedObject
# class will set object._parent to force the real parent model to be loaded
# the first time it is referenced.
yield base.DeserializedObject(object, m2m_data)
def get_model(self, name):
"""Gets the model name and fields"""
model = _get_model(name)
if model not in self.field_names_cache:
self.field_names_cache[model] = {
f.name
for f in model._meta.get_fields()
}
return (model, self.field_names_cache[model])
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
# Local imports to allow using modified versions of `_get_model`
# It could be patched in runtime via `unittest.mock.patch` for example
from django.core.serializers.python import Deserializer as PythonDeserializer, _get_model
ignore = options.pop("ignorenonexistent", False)
if not isinstance(stream_or_string, (bytes, str)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode("utf-8")
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj["model"])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in obj["fields"].items():
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(
field,
LinkedCurrencyMoneyField) and field_value is not None:
currency_field_name = get_currency_field_name(field_name,
field=field)
money_fields[field_name] = Money(
field_value,
get_currency_from_obj(Model, obj,
currency_field_name)) # noqa
elif isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj["fields"][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj["fields"] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
raise DeserializationError.with_traceback(DeserializationError(exc),
sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
object_list = simplejson.load(stream)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value,
options.get(
"encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
m2m_data[field.name] = [
m2m_convert(smart_unicode(pk)) for pk in field_value
]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
# handle a dictionary which means to lookup the instance
# and get the primary key this way (works great on
# GFK values)
if isinstance(field_value, dict):
lookup_params = {}
for k, v in field_value.iteritems():
lookup_params[k.encode("ascii")] = v
field_value = field.rel.to._default_manager.get(
**lookup_params).pk
data[field.attname] = field.rel.to._meta.get_field(
field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer_with_debugging(original_function, object_list, **options):
from django.core.serializers.python import _get_model
from django.db import DEFAULT_DB_ALIAS
from django.utils.encoding import smart_unicode
from django.conf import settings
print "loading all: %s" % object_list
db = options.pop('using', DEFAULT_DB_ALIAS)
db_models.get_apps()
for d in object_list:
print "loading %s" % d
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, db_models.ManyToManyRel):
print " field = %s" % field
print " field.rel = %s" % field.rel
print " field.rel.to = %s" % field.rel.to
print " field.rel.to._default_manager = %s" % (
field.rel.to._default_manager)
print " field.rel.to.objects = %s" % (
field.rel.to.objects)
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_unicode(field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
for i, pk in enumerate(field_value):
print " %s: converted %s to %s" % (field.name,
pk, m2m_data[field.name][i])
result = original_function(object_list, **options)
print " result = %s" % result
import traceback
traceback.print_stack()
return result
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in properties.iteritems():
if k in field_names:
fields[k] = v
obj = {"model": model_name, "pk": dictobj['id'], "fields": fields}
shape = asShape(dictobj['geometry'])
obj['geom'] = shape.wkt
return obj
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
object_list = simplejson.load(stream)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
# handle a dictionary which means to lookup the instance
# and get the primary key this way (works great on
# GFK values)
if isinstance(field_value, dict):
lookup_params = {}
for k, v in field_value.iteritems():
lookup_params[k.encode("ascii")] = v
field_value = field.rel.to._default_manager.get(**lookup_params).pk
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def dump_class_fixtures(objects, stream, **options):
"""
Generate fixture modules.
"""
# Construct and output the import rows
apps_models = {}
for identifier in [d['model'] for d in objects]:
appname = identifier.split('.')[0]
if appname not in apps_models:
apps_models[appname] = []
try:
Model = _get_model(identifier)
except DeserializationError:
raise SerializationError('Cannot resolve "%s" to a model class' % identifier)
if Model._meta.object_name not in apps_models[appname]:
apps_models[appname].append(Model._meta.object_name)
model_imports = []
for app, models in apps_models.items():
model_imports.append('from %s.models import %s' % (app, ', '.join(sorted(models))))
stream.write(
'# -*- coding: utf-8 -*-\n'
'import datetime\n'
'from decimal import Decimal\n'
'from class_fixtures.models import Fixture\n'
'%s\n\n' % '\n'.join(model_imports)
)
# Construct and output the Fixture instances
fixture_instantiations = []
for app, models in apps_models.items():
for model in models:
fixture_instantiations.append('%s_%s_fixture = Fixture(%s)' % (
app, model.lower(), model
))
stream.write('\n'.join(fixture_instantiations) + '\n\n')
# Construct and output the Fixture.add() calls in the dependency-resolved
# order that 'objects' is in. The field names are in the "fields"
# dictionary which, being a dictionary, is unordered. To make the output
# predictable and testable, we use a subclass of OrderedDict that produces
# alphabetized dict-like repr() output.
for pk, identifier, fields in [(d['pk'], d['model'], d['fields']) for d in objects]:
app, model = identifier.split('.')
kwargs = ClassicReprOrderedDict(sorted(fields.items(), key=lambda field: field[0]))
stream.write('%s_%s_fixture.add(%s, **%s)\n' % (app, model, pk, repr(kwargs)))
def PreDeserializer(objects, **options):
db = options.pop('using', DEFAULT_DB_ALIAS)
for d in objects:
Model = _get_model(d["model"])
for vfield in Model._meta.virtual_fields:
if not vfield.name in d['fields']:
continue
ct_natural_key, fk_natural_key = d['fields'][vfield.name]
ct = ContentType.objects.get_by_natural_key(*ct_natural_key)
obj = ct.model_class()._default_manager.db_manager(db).get_by_natural_key(*fk_natural_key)
d['fields'][vfield.ct_field] = ct.pk
d['fields'][vfield.fk_field] = obj.pk
del d['fields'][vfield.name]
yield d
def Deserializer(object_list, **options):
"""Deserialize simple Python objects back into Model instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = python._get_model(d["model"])
data = {}
key = resolve_key(Model._meta.module_name, d["pk"])
if key.name():
data["key_name"] = key.name()
parent = None
if key.parent():
parent = FakeParent(key.parent())
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value,
options.get(
"encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model.properties()[field_name]
if isinstance(field, db.Reference):
# Resolve foreign key references.
data[field.name] = resolve_key(Model._meta.module_name,
field_value)
if not data[field.name].name():
raise base.DeserializationError(
u"Cannot load Reference with "
"unnamed key: '%s'" % field_value)
else:
data[field.name] = field.validate(field_value)
# Create the new model instance with all it's data, but no parent.
object = Model(**data)
# Now add the parent into the hidden attribute, bypassing the type checks
# in the Model's __init__ routine.
object._parent = parent
# When the deserialized object is saved our replacement DeserializedObject
# class will set object._parent to force the real parent model to be loaded
# the first time it is referenced.
yield base.DeserializedObject(object, m2m_data)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of YAML data.
********
All this is copied from the python base deserializer but for 2 lines
********
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
object_list = yaml.load(stream)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
#These are those 2 lines ******
if isinstance(field_value, dict):
field_value = field.rel.to._default_manager.get(**field_value).pk
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
# Local imports to allow using modified versions of `_get_model`
# It could be patched in runtime via `unittest.mock.patch` for example
from django.core.serializers.python import (
Deserializer as PythonDeserializer,
_get_model,
)
ignore = options.pop('ignorenonexistent', False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj['model'])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in six.iteritems(obj['fields']):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(field_value, obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
six.reraise(DeserializationError, DeserializationError(exc), sys.exc_info()[2])
def Deserializer(object_list, **options):
"""Deserialize simple Python objects back into Model instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = python._get_model(d["model"])
data = {}
key = resolve_key(Model._meta.module_name, d["pk"])
if key.name():
data["key_name"] = key.name()
parent = None
if key.parent():
parent = FakeParent(key.parent())
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(
field_value, options.get("encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model.properties()[field_name]
if isinstance(field, db.Reference):
# Resolve foreign key references.
data[field.name] = resolve_key(Model._meta.module_name, field_value)
if not data[field.name].name():
raise base.DeserializationError(u"Cannot load Reference with "
"unnamed key: '%s'" % field_value)
else:
data[field.name] = field.validate(field_value)
# Create the new model instance with all it's data, but no parent.
object = Model(**data)
# Now add the parent into the hidden attribute, bypassing the type checks
# in the Model's __init__ routine.
object._parent = parent
# When the deserialized object is saved our replacement DeserializedObject
# class will set object._parent to force the real parent model to be loaded
# the first time it is referenced.
yield base.DeserializedObject(object, m2m_data)
def PreDeserializer(objects, **options):
db = options.pop('using', DEFAULT_DB_ALIAS)
for d in objects:
Model = _get_model(d["model"])
for vfield in Model._meta.virtual_fields:
if not isinstance(vfield, GenericForeignKey):
continue
if not vfield.name in d['fields']:
continue
ct_natural_key, fk_natural_key = d['fields'][vfield.name]
ct = ContentType.objects.get_by_natural_key(*ct_natural_key)
obj = ct.model_class()._default_manager.db_manager(
db).get_by_natural_key(*fk_natural_key)
d['fields'][vfield.ct_field] = ct.pk
d['fields'][vfield.fk_field] = obj.pk
del d['fields'][vfield.name]
yield d
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in properties.iteritems():
if k in field_names:
fields[k] = v
obj = {
"model" : model_name,
"pk" : dictobj['id'],
"fields" : fields
}
shape = asShape(dictobj['geometry'])
obj['geom'] = shape.wkt
return obj
def read_action(obj, **attrs):
act = obj.attrib['type']
s = obj.attrib['name']
if obj.attrib.get('name'):
s = _(s)
fields = {
'name': s,
}
if 'model' in obj.attrib:
model = _get_model(obj.attrib['model'])
fields['model'] = ContentType.objects.get_by_natural_key(model._meta.app_label, model._meta.model_name)
action = {
'model': act,
'id': obj.attrib['id'],
'children': [],
'fields': fields,
}
return read_object(action, **attrs)
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
ignore = options.pop('ignorenonexistent', False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj['model'])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in six.iteritems(obj['fields']):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
six.reraise(DeserializationError, DeserializationError(exc),
sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
ignore = options.pop('ignorenonexistent', False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
money_fields = {}
fields = {}
try:
Model = _get_model(obj["model"])
except DeserializationError:
if ignore:
continue
else:
raise
try:
field_names = set(f.name for f in Model._meta.get_fields())
except AttributeError:
field_names = set(f.name for f in Model._meta.fields)
for (field_name, field_value) in six.iteritems(obj['fields']):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(obj.object, field, value)
yield obj
except GeneratorExit:
raise
def read_object(obj, **attrs):
if not isinstance(obj, dict):
values = obj.getchildren()
obj = dict(obj.attrib)
else:
values = obj['children']
if 'fields' not in obj:
obj['fields'] = {}
for child in values:
if child.tag == 'field':
if 'ref' in child.attrib:
obj['fields'][child.attrib['name']] = Object.get_object(
child.attrib['ref']).object_id
elif 'model' in child.attrib:
obj['fields'][child.attrib['name']] = ContentType.objects.only(
'pk').get_by_natural_key(
*child.attrib['model'].split('.')).pk
else:
s = child.text
if 'translate' in child.attrib:
s = _(s)
obj['fields'][child.attrib['name']] = s
obj_name = obj.pop('id')
obj_id = None
try:
print('Load object', obj_name)
obj_id = Object.objects.get(name=obj_name)
instance = obj_id.content_object
for k, v in obj['fields'].items():
setattr(instance, k, v)
instance.save()
except ObjectDoesNotExist:
instance = base.build_instance(_get_model(obj['model']), obj['fields'],
attrs.get('using', DEFAULT_DB_ALIAS))
instance.save()
ct = ContentType.objects.get_by_natural_key(
instance._meta.app_label, instance._meta.model_name.lower())
obj_id = Object.objects.create(name=obj_name,
object_id=instance.pk,
content_type=ct)
return instance
def read_action(obj, **attrs):
act = obj.attrib['type']
s = obj.attrib['name']
if obj.attrib.get('name'):
s = _(s)
fields = {
'name': s,
}
if 'model' in obj.attrib:
model = _get_model(obj.attrib['model'])
fields['model'] = ContentType.objects.get_by_natural_key(
model._meta.app_label, model._meta.model_name)
action = {
'model': act,
'id': obj.attrib['id'],
'children': [],
'fields': fields,
}
return read_object(action, **attrs)
def view(request, *args, **kwargs):
with transaction.atomic():
req_payload = json_lib.loads(request.body.decode('utf-8'))
auth_info = req_payload.get('auth')
request_params = req_payload.get('request_params')
if not request_params:
raise MissingRpcData({
'error':
RPCHandler.REQUEST_DATA_MISSING,
'msg':
'No request params found in request..'
})
app_str = request_params.get('app')
model_str = request_params.get('model')
manager_str = request_params.get('manager')
procedure_str = request_params.get('procedure')
try:
model = _get_model('%s.%s' % (app_str, model_str))
except DeserializationError as e:
raise InvalidRpcRequestData('invalid app or model name')
try:
manager = getattr(model, manager_str)
except AttributeError:
raise InvalidRpcRequestData('invalid manager name')
if manager:
try:
procedure = getattr(manager, procedure_str)
except AttributeError as e:
raise InvalidRpcRequestData(
"no such procedure with name %s" % procedure_str)
res = procedure(**request_params.get(
"procedure_params")) #only named parameters calls allowed
return HttpResponse(json_lib.dumps(res),
status=200,
content_type='application/json')
raise InvalidRpcRequestData({
'error':
RPCHandler.INVALID_REQUEST_DATA,
'msg':
'Invalid or missing request data'
})
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
ignore = options.pop('ignorenonexistent', False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
money_fields = {}
fields = {}
try:
Model = _get_model(obj['model'])
except DeserializationError:
if ignore:
continue
else:
raise
try:
field_names = set(f.name for f in Model._meta.get_fields())
except AttributeError:
field_names = set(f.name for f in Model._meta.fields)
for (field_name, field_value) in six.iteritems(obj['fields']):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(field_value, obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except GeneratorExit:
raise
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = options.get("model_name") or properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in iteritems(properties):
if k in field_names:
fields[k] = v
obj = {
"model": model_name,
"pk": dictobj.get('id') or properties.get('id'),
"fields": fields
}
if asShape is None:
raise DeserializationError('shapely is not installed')
shape = asShape(dictobj['geometry'])
obj['fields'][geometry_field] = shape.wkt
return obj
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = options.get("model_name") or properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in iteritems(properties):
if k in field_names:
fields[k] = v
obj = {
"model": model_name,
"pk": dictobj.get('id') or properties.get('id'),
"fields": fields
}
if asShape is None:
raise DeserializationError('shapely is not installed')
shape = asShape(dictobj['geometry'])
obj['fields'][geometry_field] = shape.wkt
return obj
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = options.get("model_name") or properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in iteritems(properties):
if k in field_names:
fields[k] = v
obj = {
"model": model_name,
"pk": dictobj.get('id') or properties.get('id'),
"fields": fields
}
if isinstance(model._meta.get_field(geometry_field), GeoJSONField):
obj['fields'][geometry_field] = dictobj['geometry']
else:
shape = GEOSGeometry(json.dumps(dictobj['geometry']))
obj['fields'][geometry_field] = shape.wkt
return obj
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = options.get("model_name") or properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in iteritems(properties):
if k in field_names:
fields[k] = v
obj = {
"model": model_name,
"pk": dictobj.get('id') or properties.get('id'),
"fields": fields
}
if isinstance(model._meta.get_field(geometry_field), GeoJSONField):
obj['fields'][geometry_field] = dictobj['geometry']
else:
shape = GEOSGeometry(json.dumps(dictobj['geometry']))
obj['fields'][geometry_field] = shape.wkt
return obj
def read_object(obj, **attrs):
if not isinstance(obj, dict):
values = obj.getchildren()
obj = dict(obj.attrib)
else:
values = obj['children']
if 'fields' not in obj:
obj['fields'] = {}
for child in values:
if child.tag == 'field':
if 'ref' in child.attrib:
obj['fields'][child.attrib['name']] = Object.get_object(child.attrib['ref']).object_id
elif 'model' in child.attrib:
obj['fields'][child.attrib['name']] = ContentType.objects.only('pk').get_by_natural_key(*child.attrib['model'].split('.')).pk
else:
s = child.text
if 'translate' in child.attrib:
s = _(s)
obj['fields'][child.attrib['name']] = s
obj_name = obj.pop('id')
obj_id = None
try:
obj_id = Object.objects.get(name=obj_name)
instance = obj_id.content_object
for k, v in obj['fields'].items():
setattr(instance, k, v)
instance.save()
except ObjectDoesNotExist:
instance = base.build_instance(_get_model(obj['model']), obj['fields'], attrs.get('using', DEFAULT_DB_ALIAS))
instance.save()
ct = ContentType.objects.get_by_natural_key(instance._meta.app_label, instance._meta.model_name.lower())
obj_id = Object.objects.create(
name=obj_name,
object_id=instance.pk,
content_type=ct
)
return instance
def RemoteDeserializer(object_or_list, **options):
models.get_apps()
if isinstance(object_or_list, dict):
object_or_list = [ object_or_list ]
for d in object_or_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {}
if d.has_key("pk"):
data[ Model._meta.pk.attname ] = Model._meta.pk.to_python(d["pk"])
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
m2m_convert = field.rel.to._meta.pk.to_python
m2m_data[field.name] = [m2m_convert(smart_unicode(pk)) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield DeserializedObject(Model(**data), m2m_data)
def CustomDeserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
db = options.pop('using', DEFAULT_DB_ALIAS)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
filefields = []
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_unicode(field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
if hasattr(field_value, '__iter__'):
obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value)
value = getattr(obj, field.rel.field_name)
# If this is a natural foreign key to an object that
# has a FK/O2O as the foreign key, use the FK value
if field.rel.to._meta.pk.rel:
value = value.pk
else:
value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
data[field.attname] = value
else:
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
elif isinstance(field, FileField):
if field_value:
if field.name in d["changes"] or d["action"] == 1:
filefields.append([field, field_value['name'], ContentFile(base64.decodestring(field_value['data']))])
else:
filefields.append([field, field_value['name'], None])
else:
filefields.append([field, None, None])
else:
data[field.name] = field.to_python(field_value)
yield CustomDeserializedObject(Model(**data), m2m_data, filefields)
def dispatch(self, backend_configs):
for backend_cfg in backend_configs:
backend_cfg = deepcopy(backend_cfg)
try:
fetcher_source = backend_cfg.pop('fetcher')
input_name = backend_cfg.pop('input', '')
# TODO: use ContentType.objects.get_by_natural_key() ?
# TODO: accept swappable ID ?
model = _get_model(backend_cfg.pop('model'))
method = backend_cfg.pop('method', '')
subject = backend_cfg['subject']
except KeyError as e:
raise ImproperlyConfigured(
u'You have an error in your CRUDITY_BACKENDS settings. '
u'Check if "{}" is present'.format(e)) from e
except DeserializationError as de:
raise ImproperlyConfigured(de) from de
else:
backend_cls = self._backends.get(model)
if backend_cls is None:
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'no backend is registered for this model <{}>'.format(
model))
fetcher = self.get_fetcher(fetcher_source)
if fetcher is None:
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'invalid fetcher "{}".'.format(fetcher_source))
if subject == '*':
if fetcher.get_default_backend() is not None:
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'only one fallback backend is allowed for "{}/{}".'
.format(
fetcher_source,
input_name,
))
backend_cfg['source'] = fetcher_source
backend_instance = backend_cls(backend_cfg)
if not hasattr(backend_instance, 'fetcher_fallback'):
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'the backend for {} cannot be used as fallback '
u'(ie: subject="*").'.format(model))
backend_instance.fetcher_name = fetcher_source
fetcher.register_default_backend(backend_instance)
else:
if not input_name:
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'you have to declare an input for the fetcher {}.'
.format(fetcher_source))
if not method:
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'you have to declare a method for "{}/{}".'.
format(fetcher_source, input_name))
crud_input = fetcher.get_input(input_name, method)
if not crud_input:
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'invalid input "{}" for the fetcher "{}".'.format(
input_name, fetcher_source))
# TODO: move this code to backend
backend_cfg['source'] = u'{} - {}'.format(
fetcher_source, input_name)
backend_cfg[
'verbose_source'] = crud_input.verbose_name # For i18n
backend_cfg[
'verbose_method'] = crud_input.verbose_method # For i18n
backend_instance = backend_cls(backend_cfg,
crud_input=crud_input)
backend_instance.fetcher_name = fetcher_source
backend_instance.input_name = input_name
if crud_input.get_backend(backend_instance.subject):
raise ImproperlyConfigured(
u'settings.CRUDITY_BACKENDS: '
u'this (normalised) subject must be unique for "{}/{}": {}'
.format(fetcher_source, input_name,
backend_instance.subject))
crud_input.add_backend(backend_instance)
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
# Copied almost without changes from djmoney.serializers (django-money).
# Adding support for situation where old models to be deserialized have
# price field, but not price_currency field.
# In Ralph, price field existed before in various models as
# a Decimal field. All price fields were migrated to MoneyField
# without changing the original field name. This can cause problems
# in original django-money's implementation of Deserializer.
# This updated Deserializer is needed to get reversion (django-reversion)
# to work in circumstances described above.
from django.core.serializers.python import \
Deserializer as PythonDeserializer, _get_model
ignore = options.pop("ignorenonexistent", False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode("utf-8")
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj["model"])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in six.iteritems(obj["fields"]):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
try:
currency = \
obj["fields"][get_currency_field_name(field_name)]
except KeyError:
currency = DEFAULT_CURRENCY_CODE
money_fields[field_name] = Money(field_value, currency)
else:
fields[field_name] = field_value
obj["fields"] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
six.reraise(
DeserializationError, DeserializationError(exc), sys.exc_info()[2]
)
def Deserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
db = options.get('using', DEFAULT_DB_ALIAS)
ignore = options.get('ignorenonexistent', False)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
model_fields = Model._meta.get_all_field_names()
# Handle each field
for (field_name, field_value) in six.iteritems(d["fields"]):
if ignore and field_name not in model_fields:
# skip fields no longer on model
continue
if isinstance(field_value, str):
field_value = smart_text(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
field = Model._meta.get_field_by_name(field_name)[0]
# Handle M2M relations
field_rel = getattr(field, 'rel', None)
if field_rel:
field_rel_to = field_rel.to
if isinstance(field_rel, models.ManyToManyRel):
if hasattr(field_rel_to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__') and not isinstance(value, six.text_type):
return field_rel_to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_text(field_rel_to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_text(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif isinstance(field_rel, models.ManyToOneRel):
if field_value is not None:
if isinstance(field_value, dict):
#handles relations inside a dict
data[field.name] = list(Deserializer([field_value], **options))[0].object
else:
if hasattr(field_rel_to._default_manager, 'get_by_natural_key'):
if hasattr(field_value, '__iter__') and not isinstance(field_value, six.text_type):
obj = field_rel_to._default_manager.db_manager(db).get_by_natural_key(*field_value)
value = getattr(obj, field_rel.field_name)
# If this is a natural foreign key to an object that
# has a FK/O2O as the foreign key, use the FK value
if field_rel_to._meta.pk.rel:
value = value.pk
else:
value = field_rel_to._meta.get_field(field_rel.field_name).to_python(field_value)
data[field.attname] = value
else:
data[field.attname] = field_rel_to._meta.get_field(field_rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
elif hasattr(field, "get_accessor_name"):
if not field_value:
pass
elif isinstance(field_value, dict):
data[field.get_accessor_name()] = list(Deserializer([field_value], **options))[0].object
else:
raise NotImplementedError
else:
data[field.name] = field.to_python(field_value)
yield rbase.DeserializedObject(Model(**data), m2m_data)
def CustomDeserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
db = options.pop('using', DEFAULT_DB_ALIAS)
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
filefields = []
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value,
options.get(
"encoding",
settings.DEFAULT_CHARSET),
strings_only=True)
field = Model._meta.get_field(field_name)
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
if hasattr(field.rel.to._default_manager,
'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(
db).get_by_natural_key(*value).pk
else:
return smart_unicode(
field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(
field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
if hasattr(field.rel.to._default_manager,
'get_by_natural_key'):
if hasattr(field_value, '__iter__'):
obj = field.rel.to._default_manager.db_manager(
db).get_by_natural_key(*field_value)
value = getattr(obj, field.rel.field_name)
# If this is a natural foreign key to an object that
# has a FK/O2O as the foreign key, use the FK value
if field.rel.to._meta.pk.rel:
value = value.pk
else:
value = field.rel.to._meta.get_field(
field.rel.field_name).to_python(field_value)
data[field.attname] = value
else:
data[field.attname] = field.rel.to._meta.get_field(
field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
elif isinstance(field, FileField):
if field_value:
if field.name in d["changes"] or d["action"] == 1:
filefields.append([
field, field_value['name'],
ContentFile(
base64.decodestring(field_value['data']))
])
else:
filefields.append([field, field_value['name'], None])
else:
filefields.append([field, None, None])
else:
data[field.name] = field.to_python(field_value)
yield CustomDeserializedObject(Model(**data), m2m_data, filefields)
def Deserializer(object_list, **options):
"""
Deserialize simple Python objects back into Django ORM instances.
It's expected that you pass the Python objects themselves (instead of a
stream or a string) to the constructor
"""
db = options.pop('using', DEFAULT_DB_ALIAS)
#
src_version = options.pop("src_version") # version that was serialized
dest_version = options.pop("dest_version") # version that we're deserializing to
assert dest_version, "For KA Lite, we should always set the dest version to the current device."
models.get_apps()
for d in object_list:
# Look up the model and starting build a dict of data for it.
Model = _get_model(d["model"])
# See comment below for versioned fields; same logic
# applies here as well.
if hasattr(Model, "version"):
v_diff = version_diff(Model.minversion, dest_version)
if v_diff > 0 or v_diff is None:
continue
data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])}
m2m_data = {}
# Handle each field
for (field_name, field_value) in d["fields"].iteritems():
if isinstance(field_value, str):
field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True)
try:
field = Model._meta.get_field(field_name)
except models.FieldDoesNotExist as fdne:
# If src version is newer than dest version,
# or if it's unknown, then assume that the field
# is a new one and skip it.
# We can't know for sure, because
# we don't have that field (we are the dest!),
# so we don't know what version it came in on.
v_diff = version_diff(src_version, dest_version)
if v_diff > 0 or v_diff is None:
continue
# Something else must be going on, so re-raise.
else:
raise fdne
# Handle M2M relations
if field.rel and isinstance(field.rel, models.ManyToManyRel):
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
def m2m_convert(value):
if hasattr(value, '__iter__'):
return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk
else:
return smart_unicode(field.rel.to._meta.pk.to_python(value))
else:
m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v))
m2m_data[field.name] = [m2m_convert(pk) for pk in field_value]
# Handle FK fields
elif field.rel and isinstance(field.rel, models.ManyToOneRel):
if field_value is not None:
if hasattr(field.rel.to._default_manager, 'get_by_natural_key'):
if hasattr(field_value, '__iter__'):
obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value)
value = getattr(obj, field.rel.field_name)
# If this is a natural foreign key to an object that
# has a FK/O2O as the foreign key, use the FK value
if field.rel.to._meta.pk.rel:
value = value.pk
else:
value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
data[field.attname] = value
else:
data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value)
else:
data[field.attname] = None
# Handle all other fields
else:
data[field.name] = field.to_python(field_value)
yield base.DeserializedObject(Model(**data), m2m_data)
