def load_data_for_db(db_alias, objects):
"""
:param db_alias: Django alias for database to load objects into
:param objects: List of object dictionaries to load
:return: LoadStats object
"""
connection = connections[db_alias]
model_counter = Counter()
with connection.constraint_checks_disabled():
for obj in PythonDeserializer(objects, using=db_alias):
if router.allow_migrate_model(db_alias, obj.object.__class__):
model_counter.update([obj.object.__class__])
try:
obj.save(using=db_alias)
except (DatabaseError, IntegrityError) as e:
e.args = ("Could not load %(app_label)s.%(object_name)s(pk=%(pk)s): %(error_msg)s" % {
'app_label': obj.object._meta.app_label,
'object_name': obj.object._meta.object_name,
'pk': obj.object.pk,
'error_msg': force_text(e)
},)
raise
# Since we disabled constraint checks, we must manually check for
# any invalid keys that might have been added
table_names = [model._meta.db_table for model in model_counter]
try:
connection.check_constraints(table_names=table_names)
except Exception as e:
e.args = ("Problem loading data: %s" % e,)
raise
return LoadStat(db_alias, model_counter)
def load_data_for_db(db_alias):
"""
A coroutine that is sent object dictionaries and loads them into the
database identified by ``db_alias``. When it is terminated with
``None``, it yields a LoadStat object.
"""
model_counter = Counter()
with transaction.atomic(using=db_alias), \
constraint_checks_deferred(db_alias):
while True:
obj_dict = yield
if obj_dict is None:
break
for obj in PythonDeserializer([obj_dict], using=db_alias):
Model = type(obj.object)
if not router.allow_migrate_model(db_alias, Model):
continue
model_counter.update([Model])
try:
obj.save(using=db_alias)
except DatabaseError as err:
m = Model._meta
raise type(err)(
f'Could not load {m.app_label}.{m.object_name}'
f'(pk={obj.object.pk}) in DB {db_alias!r}') from err
print(f'Loading DB {db_alias!r} complete')
yield LoadStat(db_alias, model_counter)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
money_fields = {}
fields = {}
Model = _get_model(obj["model"])
for (field_name, field_value) in six.iteritems(obj['fields']):
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(obj.object, field, value)
yield obj
except GeneratorExit:
raise
def load_data_for_db(db_alias):
"""
A coroutine that is sent object dictionaries and loads them into the
database identified by ``db_alias``. When it is terminated with
``None``, it yields a LoadStat object.
"""
model_counter = Counter()
with transaction.atomic(using=db_alias), \
constraint_checks_deferred(db_alias):
while True:
obj_dict = yield
if obj_dict is None:
break
for obj in PythonDeserializer([obj_dict], using=db_alias):
Model = type(obj.object)
if not router.allow_migrate_model(db_alias, Model):
continue
model_counter.update([Model])
try:
# Force insert here to prevent Django from attempting to do an update.
# We want to ensure that if there is already data in the DB that we don't
# save over it and rather error out.
obj.save(using=db_alias, force_insert=True)
except DatabaseError as err:
logger.exception("Error saving data")
m = Model._meta
key = f"pk={obj.object.pk}"
if hasattr(obj.object, "natural_key"):
key = f"key={obj.object.natural_key()}"
raise type(err)(
f'Could not load {m.app_label}.{m.object_name}'
f'({key}) in DB {db_alias!r}') from err
print(f'Loading DB {db_alias!r} complete')
yield LoadStat(db_alias, model_counter)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in properties.iteritems():
if k in field_names:
fields[k] = v
obj = {"model": model_name, "pk": dictobj['id'], "fields": fields}
shape = asShape(dictobj['geometry'])
obj['geom'] = shape.wkt
return obj
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
try:
collection = simplejson.load(stream)
objects = map(FeatureToPython, collection['features'])
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception, e:
# Map to deserializer error
raise DeserializationError(e)
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
# Local imports to allow using modified versions of `_get_model`
# It could be patched in runtime via `unittest.mock.patch` for example
from django.core.serializers.python import Deserializer as PythonDeserializer, _get_model
ignore = options.pop("ignorenonexistent", False)
if not isinstance(stream_or_string, (bytes, str)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode("utf-8")
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj["model"])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in obj["fields"].items():
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(
field,
LinkedCurrencyMoneyField) and field_value is not None:
currency_field_name = get_currency_field_name(field_name,
field=field)
money_fields[field_name] = Money(
field_value,
get_currency_from_obj(Model, obj,
currency_field_name)) # noqa
elif isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj["fields"][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj["fields"] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
raise DeserializationError.with_traceback(DeserializationError(exc),
sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of YAML data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
for obj in PythonDeserializer(yaml.safe_load(stream), **options):
yield obj
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
for obj in PythonDeserializer(simplejson.load(stream), **options):
yield obj
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of CSV data.
"""
def process_item(item):
m = _LIST_RE.match(item)
if m:
contents = m.group(1)
if not contents:
item = []
else:
item = process_m2m(contents)
else:
if item == 'TRUE':
item = True
elif item == 'FALSE':
item = False
elif item == 'NULL':
item = None
elif (item in _QUOTED_BOOL_NULL or _QUOTED_LIST_RE.match(item)):
item = item.strip('\'"')
return item
def process_m2m(contents):
li = []
if _NK_LIST_RE.match(contents):
for item in _NK_SPLIT_RE.split(contents):
li.append(process_item(item))
else:
li = _SPLIT_RE.split(contents)
return li
if isinstance(stream_or_string, six.string_types):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
reader = UnicodeReader(stream)
header = next(reader) # first line must be a header
data = []
for row in reader:
# Need to account for the presence of multiple headers in
# the stream since serialized data can contain them.
if row[:2] == ['pk', 'model']:
# Not the best check. Perhaps csv.Sniffer.has_header
# would be better?
header = row
continue
d = dict(zip(header[:2], row[:2]))
d['fields'] = dict(zip(header[2:], map(process_item, row[2:])))
data.append(d)
for obj in PythonDeserializer(data, **options):
yield obj
def Deserializer(stream_or_string, **options):
"""Deserialize a stream or string of JSON data."""
if not isinstance(stream_or_string, (bytes, str)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode()
try:
objects = json.loads(stream_or_string)
yield from PythonDeserializer(objects, **options)
except Exception as exc:
raise DeserializationError() from exc
def Deserializer(stream_or_string, **options):
"""Deserialize a stream or string of YAML data."""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode()
if isinstance(stream_or_string, str):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
try:
yield from PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options)
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
raise DeserializationError() from exc
def Deserializer(stream_or_string, **options):
if isinstance(stream_or_string, six.string_types):
stream_or_string = six.BytesIO(stream_or_string.encode('utf-8'))
try:
objects = ijson.items(stream_or_string, 'item')
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
six.reraise(DeserializationError, DeserializationError(e),
sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, basestring):
stream = BytesIO(stream_or_string)
else:
stream = stream_or_string
try:
for obj in PythonDeserializer(json.load(stream), **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
raise DeserializationError(e)
def Deserializer(stream_or_string, **options):
"""Deserialize a stream or string of JSON data."""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode()
if isinstance(stream_or_string, (bytes, str)):
stream_or_string = stream_or_string.split("\n")
for line in stream_or_string:
if not line.strip():
continue
try:
yield from PythonDeserializer([json.loads(line)], **options)
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
raise DeserializationError() from exc
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of YAML data.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
try:
for obj in PythonDeserializer(yaml.safe_load(stream), **options):
yield obj
except GeneratorExit:
raise
except Exception, e:
# Map to deserializer error
raise DeserializationError(e)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
objects = json.loads(stream_or_string)
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
if isinstance(stream_or_string, basestring):
objects = json.loads(stream_or_string)
else:
objects = json.load(stream_or_string)
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
raise DeserializationError(e)
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of YAML data.
"""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
if isinstance(stream_or_string, six.string_types):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
try:
for obj in PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
raise DeserializationError(e)
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
ignore = options.pop('ignorenonexistent', False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj['model'])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in six.iteritems(obj['fields']):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
six.reraise(DeserializationError, DeserializationError(exc),
sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
ignore = options.pop('ignorenonexistent', False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
for obj in json.loads(stream_or_string):
money_fields = {}
fields = {}
try:
Model = _get_model(obj["model"])
except DeserializationError:
if ignore:
continue
else:
raise
try:
field_names = set(f.name for f in Model._meta.get_fields())
except AttributeError:
field_names = set(f.name for f in Model._meta.fields)
for (field_name, field_value) in six.iteritems(obj['fields']):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
money_fields[field_name] = Money(
field_value,
obj['fields'][get_currency_field_name(field_name)])
else:
fields[field_name] = field_value
obj['fields'] = fields
for obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(obj.object, field, value)
yield obj
except GeneratorExit:
raise
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
geometry_field = options.get("geometry_field", "geom")
def FeatureToPython(dictobj):
properties = dictobj['properties']
model_name = options.get("model_name") or properties.pop('model')
# Deserialize concrete fields only (bypass dynamic properties)
model = _get_model(model_name)
field_names = [f.name for f in model._meta.fields]
fields = {}
for k, v in iteritems(properties):
if k in field_names:
fields[k] = v
obj = {
"model": model_name,
"pk": dictobj.get('id') or properties.get('id'),
"fields": fields
}
if isinstance(model._meta.get_field(geometry_field), GeoJSONField):
obj['fields'][geometry_field] = dictobj['geometry']
else:
shape = GEOSGeometry(json.dumps(dictobj['geometry']))
obj['fields'][geometry_field] = shape.wkt
return obj
if isinstance(stream_or_string, string_types):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
try:
collection = json.load(stream)
objects = [FeatureToPython(f) for f in collection['features']]
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
raise DeserializationError(repr(e))
def Deserializer(stream, **options):
"""
Deserialize a stream of JSON data using iterative ijson so we may not load the whole string into memory.
"""
if isinstance(stream, (bytes, six.string_types)):
raise TypeError(
'Use iloaddata/ijson with streams only. For strings use plain loaddata/json.loads'
)
try:
objects = ijson.items(stream, 'item')
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
six.reraise(DeserializationError, DeserializationError(e),
sys.exc_info()[2])
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of JSON data.
"""
def GEOJsonToEWKT(dict):
"""
Convert to a string that GEOSGeometry class constructor can accept.
The default decoder would pass our geo dict object to the constructor which
would result in a TypeError; using the below hook we are forcing it into a
ewkt format. This is accomplished with a class hint as per JSON-RPC
"""
if '__GEOSGeometry__' in dict: # using class hint catch a GEOSGeometry definition
return dict['__GEOSGeometry__'][1][0]
return dict
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
for obj in PythonDeserializer(simplejson.load(stream, object_hook=GEOJsonToEWKT), **options):
yield obj
def Deserializer(stream_or_string, **options):
"""
Deserialize a stream or string of YAML data,
as written by the Serializer above.
"""
if isinstance(stream_or_string, basestring):
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
# Reconstruct the flat object list as PythonDeserializer expects
# NOTE: This could choke on large data sets, since it
# constructs the flattened data list in memory
data = []
for model, objects in yaml.load(stream).iteritems():
# Add the model name back into each object dict
for pk, fields in objects.iteritems():
data.append({'model': model, 'pk': pk, 'fields': fields})
# Deserialize the flattened data
for obj in PythonDeserializer(data, **options):
yield obj
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
return super(PythonSerializer, self).getvalue()
def Deserializer(stream_or_string, **options):
<<<<<<< HEAD
"""
Deserialize a stream or string of JSON data.
"""
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
try:
objects = json.loads(stream_or_string)
for obj in PythonDeserializer(objects, **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
=======
"""Deserialize a stream or string of JSON data."""
if not isinstance(stream_or_string, (bytes, str)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode()
try:
objects = json.loads(stream_or_string)
yield from PythonDeserializer(objects, **options)
def Deserializer(stream_or_string, **options): # noqa
"""
Deserialize a stream or string of JSON data.
"""
# Copied almost without changes from djmoney.serializers (django-money).
# Adding support for situation where old models to be deserialized have
# price field, but not price_currency field.
# In Ralph, price field existed before in various models as
# a Decimal field. All price fields were migrated to MoneyField
# without changing the original field name. This can cause problems
# in original django-money's implementation of Deserializer.
# This updated Deserializer is needed to get reversion (django-reversion)
# to work in circumstances described above.
from django.core.serializers.python import \
Deserializer as PythonDeserializer, _get_model
ignore = options.pop("ignorenonexistent", False)
if not isinstance(stream_or_string, (bytes, six.string_types)):
stream_or_string = stream_or_string.read()
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode("utf-8")
try:
for obj in json.loads(stream_or_string):
try:
Model = _get_model(obj["model"])
except DeserializationError:
if ignore:
continue
else:
raise
money_fields = {}
fields = {}
field_names = {field.name for field in Model._meta.get_fields()}
for (field_name, field_value) in six.iteritems(obj["fields"]):
if ignore and field_name not in field_names:
# skip fields no longer on model
continue
field = Model._meta.get_field(field_name)
if isinstance(field, MoneyField) and field_value is not None:
try:
currency = \
obj["fields"][get_currency_field_name(field_name)]
except KeyError:
currency = DEFAULT_CURRENCY_CODE
money_fields[field_name] = Money(field_value, currency)
else:
fields[field_name] = field_value
obj["fields"] = fields
for inner_obj in PythonDeserializer([obj], **options):
for field, value in money_fields.items():
setattr(inner_obj.object, field, value)
yield inner_obj
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
six.reraise(
DeserializationError, DeserializationError(exc), sys.exc_info()[2]
)
Deserialize a stream or string of YAML data.
"""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode('utf-8')
if isinstance(stream_or_string, six.string_types):
=======
"""Deserialize a stream or string of YAML data."""
if isinstance(stream_or_string, bytes):
stream_or_string = stream_or_string.decode()
if isinstance(stream_or_string, str):
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
stream = StringIO(stream_or_string)
else:
stream = stream_or_string
try:
<<<<<<< HEAD
for obj in PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options):
yield obj
except GeneratorExit:
raise
except Exception as e:
# Map to deserializer error
six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
=======
yield from PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options)
except (GeneratorExit, DeserializationError):
raise
except Exception as exc:
raise DeserializationError() from exc
>>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
def handle_model(self, m):
"""Called to handle a model"""
a = []
for obj in PythonDeserializer([m],**self.options.copy()):
a.append(obj)
return a