def __next__(self): if not self._current_row_is_valid(): """ If the current row is not valid, next will try to go to the next sheet and deserialize the first object. """ try: self._select_next_sheet() except: raise StopIteration # Build object: if self._current_row_is_valid(): values = self._row_to_dict(self.workbook.active[self.current_row], self.fields) obj = base.build_instance(self.model_class, values, False) obj.save() else: # Sheets without data (only with header): raise StopIteration # Updata row position: self.current_row += 1 return base.DeserializedObject(obj, {})
def Deserializer(object_list, **options): """Deserialize simple Python objects back into Model instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = python._get_model(d["model"]) data = {} key = resolve_key(Model._meta.module_name, d["pk"]) if key.name(): data["key_name"] = key.name() parent = None if key.parent(): parent = FakeParent(key.parent()) m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get( "encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model.properties()[field_name] if isinstance(field, db.Reference): # Resolve foreign key references. data[field.name] = resolve_key(Model._meta.module_name, field_value) else: # Handle converting strings to more specific formats. if isinstance(field_value, basestring): if isinstance(field, db.DateProperty): field_value = datetime.datetime.strptime( field_value, '%Y-%m-%d').date() elif isinstance(field, db.TimeProperty): field_value = parse_datetime_with_microseconds( field_value, '%H:%M:%S').time() elif isinstance(field, db.DateTimeProperty): field_value = parse_datetime_with_microseconds( field_value, '%Y-%m-%d %H:%M:%S') # Handle pyyaml datetime.time deserialization - it returns a datetime # instead of a time. if (isinstance(field_value, datetime.datetime) and isinstance(field, db.TimeProperty)): field_value = field_value.time() data[field.name] = field.validate(field_value) # Create the new model instance with all it's data, but no parent. object = Model(**data) # Now add the parent into the hidden attribute, bypassing the type checks # in the Model's __init__ routine. object._parent = parent # When the deserialized object is saved our replacement DeserializedObject # class will set object._parent to force the real parent model to be loaded # the first time it is referenced. yield base.DeserializedObject(object, m2m_data)
def load_block(self, m_name, pk, data): """ Import a single block of fields into the given model. """ if self.deferred_models[m_name] > 3: # Model is deferred! return None try: model, fields = self.get_model(m_name) except base.DeserializationError: sys.stderr.write("\nSkipping all items from model {m_name}!\n") self.deferred_models[m_name] = 10 return None try: data = self.build_data(model, fields, pk, data) m2m_fields = data.pop('m2m', {}) except (base.M2MDeserializationError, base.DeserializationError, ObjectDoesNotExist): return None try: obj = base.build_instance(model, data, None) return base.DeserializedObject(obj, m2m_fields, {}) except Exception: return None # Any error, defer the object.
def _handle_object(self, node): """ Convert a model node to a DeserializedObject. """ # Look up the model using the model loading mechanism. If this fails, # bail. Model = self._get_model_from_node(node) # Start building a data dictionary from the object. # If the node is missing the pk set it to None if node.hasAttribute("hotjazz:pk"): pk = node.getAttribute("hotjazz:pk") else: pk = None data = {Model._meta.pk.attname: Model._meta.pk.to_python(pk)} # Also start building a dict of m2m data (this is saved as # {m2m_accessor_attribute : [list_of_related_objects]}) m2m_data = {} # Deseralize each field. for child_node in node.childNodes: # ignore anything but elements if child_node.nodeType != Node.ELEMENT_NODE: continue # if the object has non-field contents, bail if not child_node.hasAttribute("hotjazz:type") \ or not child_node.getAttribute("hotjazz:type").endswith("Field"): raise base.DeserializationError("Unrecognized node (%s)" % smart_unicode(child_node)) field_node = child_node field_name = field_node.tagName.split(":")[-1] # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will # be propagated correctly. field = Model._meta.get_field(field_name) # As is usually the case, relation fields get the special treatment. if field.rel and isinstance(field.rel, models.ManyToManyRel): m2m_data[field.name] = self._handle_m2m_field_node( field_node, field) elif field.rel and isinstance(field.rel, models.ManyToOneRel): data[field.attname] = self._handle_fk_field_node( field_node, field) else: value = None try: value = field.to_python(getInnerText(field_node).strip()) except ValidationError: # assuming this is a NULL value for a non-char column pass data[field.name] = value # Return a DeserializedObject so that the m2m data has a place to live. result = base.DeserializedObject(Model(**data), m2m_data) return result
def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__'): return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk else: return smart_unicode(field.rel.to._meta.pk.to_python(value)) else: m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v)) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__'): obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value) value = getattr(obj, field.rel.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if field.rel.to._meta.pk.rel: value = value.pk else: value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string object_list = simplejson.load(stream) models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get( "encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): m2m_convert = field.rel.to._meta.pk.to_python m2m_data[field.name] = [ m2m_convert(smart_unicode(pk)) for pk in field_value ] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: # handle a dictionary which means to lookup the instance # and get the primary key this way (works great on # GFK values) if isinstance(field_value, dict): lookup_params = {} for k, v in field_value.iteritems(): lookup_params[k.encode("ascii")] = v field_value = field.rel.to._default_manager.get( **lookup_params).pk data[field.attname] = field.rel.to._meta.get_field( field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def _handle_object(self, node): """ Convert an <object> node to a DeserializedObject. """ # Look up the model using the model loading mechanism. If this fails, # bail. Model = self._get_model_from_node(node, "model") # Start building a data dictionary from the object. If the node is # missing the pk attribute, bail. pk = node.getAttribute("pk") if not pk: raise base.DeserializationError( "<object> node is missing the 'pk' attribute") data = {Model._meta.pk.attname: Model._meta.pk.to_python(pk)} # Also start building a dict of m2m data (this is saved as # {m2m_accessor_attribute : [list_of_related_objects]}) m2m_data = {} # Deseralize each field. for field_node in node.getElementsByTagName("field"): # If the field is missing the name attribute, bail (are you # sensing a pattern here?) field_name = field_node.getAttribute("name") if not field_name: raise base.DeserializationError( "<field> node is missing the 'name' attribute") # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will # be propagated correctly. field = Model._meta.get_field(field_name) # As is usually the case, relation fields get the special treatment. if field.rel and isinstance(field.rel, models.ManyToManyRel): m2m_data[field.name] = self._handle_m2m_field_node( field_node, field) elif field.rel and isinstance(field.rel, models.ManyToOneRel): data[field.attname] = self._handle_fk_field_node( field_node, field) else: if field_node.getElementsByTagName('None'): value = None elif isinstance(field, TransField): value = field.to_python( xml_serializer.getInnerText( field_node).strip()).raw_data else: value = field.to_python( xml_serializer.getInnerText(field_node).strip()) data[field.name] = value # Return a DeserializedObject so that the m2m data has a place to live. return base.DeserializedObject(Model(**data), m2m_data)
def _handle_object(self, node): """ Convert an <object> node to a DeserializedObject. """ # Look up the model using the model loading mechanism. If this fails, # bail. Model = self._get_model_from_node(node, "model") # Start building a data dictionary from the object. data = {} if node.hasAttribute('pk'): data[Model._meta.pk.attname] = Model._meta.pk.to_python( node.getAttribute('pk')) # Also start building a dict of m2m data (this is saved as # {m2m_accessor_attribute : [list_of_related_objects]}) m2m_data = {} field_names = {f.name for f in Model._meta.get_fields()} # Deserialize each field. for field_node in node.getElementsByTagName("field"): # If the field is missing the name attribute, bail (are you # sensing a pattern here?) field_name = field_node.getAttribute("name") if not field_name: raise base.DeserializationError( "<field> node is missing the 'name' attribute") # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will # be propagated correctly unless ignorenonexistent=True is used. if self.ignore and field_name not in field_names: continue field = Model._meta.get_field(field_name) # As is usually the case, relation fields get the special treatment. if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): m2m_data[field.name] = self._handle_m2m_field_node( field_node, field) elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): data[field.attname] = self._handle_fk_field_node( field_node, field) else: if field_node.getElementsByTagName('None'): value = None else: value = field.to_python(getInnerText(field_node).strip()) data[field.name] = value obj = base.build_instance(Model, data, self.db) # Return a DeserializedObject so that the m2m data has a place to live. return base.DeserializedObject(obj, m2m_data)
def Deserializer(file, **options): ''' Execs the python fixture, and returns an iterator of objects that loaddata can save. Pretty dangerous; this may be why this functionality is not built-in to Django. ''' exec file #objects = locals().get('objects') # even if these are not strictly in order, the sub-references will be saved on demand objects = [ v for v in locals().values() if isinstance(v, django_models.Model) ] for obj in objects: yield base.DeserializedObject(obj, getattr(obj, 'm2m_data', {}))
def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) data = {Model._meta.pk.attname: Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, unicode): field_value = field_value.encode( options.get("encoding", settings.DEFAULT_CHARSET)) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): pks = [] m2m_convert = field.rel.to._meta.pk.to_python for pk in field_value: if isinstance(pk, unicode): pks.append( m2m_convert( pk.encode( options.get("encoding", settings.DEFAULT_CHARSET)))) else: pks.append(m2m_convert(pk)) m2m_data[field.name] = pks # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value: data[field.attname] = field.rel.to._meta.get_field( field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def _handle_object(self, node): """Convert an <object> node to a DeserializedObject.""" # Look up the model using the model loading mechanism. If this fails, # bail. Model = self._get_model_from_node(node, "model") # Start building a data dictionary from the object. data = {} field_names = {f.name for f in Model._meta.get_fields()} # Do the pk #obj_pk = obj.pk # if obj_pk is not None: # data['pk'] = str(obj_pk) for field_node in node.getElementsByTagName("field"): field_name = field_node.getAttribute("name") # If the field is missing the name attribute, bail if not field_name: raise base.DeserializationError("<field> node is missing the 'name' attribute") # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will # be propagated correctly unless ignorenonexistent or map is used. if self.ignore and field_name not in field_names: continue field = Model._meta.get_field(field_name) # Relation fields error or ignore if (field.remote_field and (isinstance(field.remote_field, models.ManyToManyRel) or isinstance(field.remote_field, models.ManyToOneRel)) ): if self.ignore: continue else: if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): raise base.DeserializationError("<field> node is a (not parsable) Many to Many field") else: raise base.DeserializationError("<field> node is a (not parsable) Many to One foreign key field") else: #print('node:' + str(field_node.childNodes)) c = field_node.firstChild if (c): value = field.to_python(c.data.strip()) else: value = None data[field.name] = value obj = Model(**data) return base.DeserializedObject(obj)
def Deserializer(object_list, **options): """Deserialize simple Python objects back into Model instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = python._get_model(d["model"]) data = {} key = resolve_key(Model._meta.module_name, d["pk"]) if key.name(): data["key_name"] = key.name() parent = None if key.parent(): parent = FakeParent(key.parent()) m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get( "encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model.properties()[field_name] if isinstance(field, db.Reference): # Resolve foreign key references. data[field.name] = resolve_key(Model._meta.module_name, field_value) if not data[field.name].name(): raise base.DeserializationError( u"Cannot load Reference with " "unnamed key: '%s'" % field_value) else: data[field.name] = field.validate(field_value) # Create the new model instance with all it's data, but no parent. object = Model(**data) # Now add the parent into the hidden attribute, bypassing the type checks # in the Model's __init__ routine. object._parent = parent # When the deserialized object is saved our replacement DeserializedObject # class will set object._parent to force the real parent model to be loaded # the first time it is referenced. yield base.DeserializedObject(object, m2m_data)
def __next__(self): d = self.data_it.__next__() # Look up the model using the model loading mechanism. model_path = None try: model_path = self.model_path_from_data(d) model_class = self.get_model_class(model_path) except base.DeserializationError: if self.ignore: return self.__next__() else: raise # Start building a data dictionary from the object. data = {} pk = self.get_pk_from_data(d) if (pk): try: data[model_class._meta.pk.attname] = self.pk_to_python(model_class, pk) except Exception as e: raise base.DeserializationError.WithData(e, model_path, pk, None) if model_class not in self.field_names_cache: self.field_names_cache[model_class] = self.field_names(model_class) field_names = self.field_names_cache[model_class] # Handle each field for (field_name, field_value) in self.fields_from_data(d).items(): if self.ignore and field_name not in field_names: continue field = model_class._meta.get_field(field_name) # Do not handle relation fields. if(self.field_is_nonrelational(self.ignore, model_class, field)): try: data[field.name] = field.to_python(field_value) except Exception as e: raise base.DeserializationError("{}: ({}:pk={}) field:'{}': field_value:'{}'".format( e, model_path, pk, field_name, field_value )) obj = base.build_instance(model_class, data, self.db) return base.DeserializedObject(obj)
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if isinstance(stream_or_string, str): stream = StringIO(stream_or_string) else: stream = stream_or_string models.get_apps() object_list = simplejson.load(stream) if not isinstance(object_list, list): object_list = [object_list] for obj in object_list: # Look up the model and starting build a dict of data for it. if 'screen_name' in obj: Model = _get_model('twitter_roa.user') else: Model = _get_model("twitter_roa.tweet") data = {} m2m_data = {} # Handle each field for (field_name, field_value) in obj.items(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get( "encoding", DEFAULT_CHARSET), strings_only=True) try: field = Model._meta.get_field(field_name) except models.FieldDoesNotExist: continue # Handle FK fields if field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: data[field.attname] = field.rel.to._meta.\ get_field(field.rel.field_name).\ to_python(field_value['id']) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def __next__(self): if self.current_row < self.num_objects + 3: values = {} for ix in range(self.num_fields): values[self.fields[ix][0]] = self.get_value( self.ws[self.current_row][ix], self.fields[ix][1]) present = datetime.now() # The following is not necessary since we are saving the object using it's save method # However, it doesn't hurt for af in self.auto_now_fields: if af.name not in values or values[af.name] is None: values[af.name] = present obj = base.build_instance(self.Model, values, False) self.current_row += 1 obj.save() return base.DeserializedObject(obj, {}) raise StopIteration
def _handle_object(self, node): """Convert an <object> node to a DeserializedObject.""" # Look up the model using the model loading mechanism. If this fails, # bail. model_path = node.getAttribute("model") model_class = self.get_model_class(model_path) # Start building a data dictionary from the object. data = {} if node.hasAttribute('pk'): data[model_class._meta.pk.attname] = self.pk_to_python(model_class, node.getAttribute('pk')) field_names = self.field_names(model_class) # Deserialize each field. for field_node in node.getElementsByTagName("field"): # If the field is missing the name attribute, bail field_name = field_node.getAttribute("name") if not field_name: raise base.DeserializationError("<field> node is missing the 'name' attribute") # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will # be propagated correctly unless ignorenonexistent=True is used. if self.ignore and field_name not in field_names: continue field = model_class._meta.get_field(field_name) # Do not handle relation fields. if(self.field_is_nonrelational(self.ignore, model_class, field)): if field_node.getElementsByTagName('None'): value = None else: value = field.to_python(getInnerText(field_node).strip()) data[field.name] = value obj = base.build_instance(model_class, data, self.db) return base.DeserializedObject(obj)
def Deserializer(object_list, *, using=DEFAULT_DB_ALIAS, ignorenonexistent=False, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ handle_forward_references = options.pop("handle_forward_references", False) field_names_cache = {} # Model: <list of field_names> for d in object_list: # Look up the model and starting build a dict of data for it. try: Model = _get_model(d["model"]) except base.DeserializationError: if ignorenonexistent: continue else: raise data = {} if "pk" in d: try: data[Model._meta.pk.attname] = Model._meta.pk.to_python( d.get("pk")) except Exception as e: raise base.DeserializationError.WithData( e, d["model"], d.get("pk"), None) m2m_data = {} deferred_fields = {} if Model not in field_names_cache: field_names_cache[Model] = { f.name for f in Model._meta.get_fields() } field_names = field_names_cache[Model] # Handle each field for (field_name, field_value) in d["fields"].items(): if ignorenonexistent and field_name not in field_names: # skip fields no longer on model continue field = Model._meta.get_field(field_name) # Handle M2M relations if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): try: values = base.deserialize_m2m_values( field, field_value, using, handle_forward_references) except base.M2MDeserializationError as e: raise base.DeserializationError.WithData( e.original_exc, d["model"], d.get("pk"), e.pk) if values == base.DEFER_FIELD: deferred_fields[field] = field_value else: m2m_data[field.name] = values # Handle FK fields elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): try: value = base.deserialize_fk_value( field, field_value, using, handle_forward_references) except Exception as e: raise base.DeserializationError.WithData( e, d["model"], d.get("pk"), field_value) if value == base.DEFER_FIELD: deferred_fields[field] = field_value else: data[field.attname] = value # Handle all other fields else: try: data[field.name] = field.to_python(field_value) except Exception as e: raise base.DeserializationError.WithData( e, d["model"], d.get("pk"), field_value) obj = base.build_instance(Model, data, using) yield base.DeserializedObject(obj, m2m_data, deferred_fields)
def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) # src_version = options.pop("src_version") # version that was serialized dest_version = options.pop("dest_version") # version that we're deserializing to assert dest_version, "For KA Lite, we should always set the dest version to the current device." models.get_apps() for d in object_list: # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) # See comment below for versioned fields; same logic # applies here as well. if hasattr(Model, "version"): v_diff = version_diff(Model.minversion, dest_version) if v_diff > 0 or v_diff is None: continue data = {Model._meta.pk.attname : Model._meta.pk.to_python(d["pk"])} m2m_data = {} # Handle each field for (field_name, field_value) in d["fields"].iteritems(): if isinstance(field_value, str): field_value = smart_unicode(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) try: field = Model._meta.get_field(field_name) except models.FieldDoesNotExist as fdne: # If src version is newer than dest version, # or if it's unknown, then assume that the field # is a new one and skip it. # We can't know for sure, because # we don't have that field (we are the dest!), # so we don't know what version it came in on. v_diff = version_diff(src_version, dest_version) if v_diff > 0 or v_diff is None: continue # Something else must be going on, so re-raise. else: raise fdne # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__'): return field.rel.to._default_manager.db_manager(db).get_by_natural_key(*value).pk else: return smart_unicode(field.rel.to._meta.pk.to_python(value)) else: m2m_convert = lambda v: smart_unicode(field.rel.to._meta.pk.to_python(v)) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__'): obj = field.rel.to._default_manager.db_manager(db).get_by_natural_key(*field_value) value = getattr(obj, field.rel.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if field.rel.to._meta.pk.rel: value = value.pk else: value = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = field.rel.to._meta.get_field(field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) yield base.DeserializedObject(Model(**data), m2m_data)
def Deserializer(manifest, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) ignore = options.pop('ignorenonexistent', False) field_names_cache = {} # Model: <list of field_names> from aristotle_mdr.models import RegistrationAuthority for ra in manifest.get('registration_authorities', []): if ra['uuid']: ra = RegistrationAuthority.objects.update_or_create( uuid=ra['uuid'], defaults={ 'name': ra['name'], 'definition': ra['definition'], }) else: ra = RegistrationAuthority.objects.create( # uuid=ra['uuid'], name=ra['name'], definition=ra['definition']) from aristotle_mdr.models import Organization for org in manifest.get('organizations', []): o, _ = Organization.objects.get_or_create( name=org['name'], definition=org['definition'], uuid=org['uuid'], ) if 'aristotle_mdr.contrib.identifiers' in settings.INSTALLED_APPS: from aristotle_mdr.contrib.identifiers.models import Namespace for namespace in org['namespaces']: Namespace.objects.get_or_create( naming_authority=o, shorthand_prefix=namespace['shorthand_prefix']) for d in manifest['metadata']: # Look up the model and starting build a dict of data for it. try: from django.contrib.contenttypes.models import ContentType Model = ContentType.objects.get( app_label=d["concept_type"]["app"], model=d["concept_type"]["model"]).model_class() except base.DeserializationError: if ignore: continue else: raise data = {} if 'pk' in d: try: data[Model._meta.pk.attname] = Model._meta.pk.to_python( d.get('pk')) except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), None) m2m_data = {} if Model not in field_names_cache: field_names_cache[Model] = { f.name for f in Model._meta.get_fields() } field_names = field_names_cache[Model] # Handle each field for (field_name, field_value) in six.iteritems(d["fields"]): if ignore and field_name not in field_names: # skip fields no longer on model continue if isinstance(field_value, str): field_value = force_text(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) if field_name in dict(getattr(Model, 'serialize_weak_entities', {})).keys(): pass # Wait else: field = Model._meta.get_field(field_name) # Handle M2M relations if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): model = field.remote_field.model if hasattr(model._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__') and not isinstance( value, six.text_type): return model._default_manager.db_manager( db).get_by_natural_key(*value).pk else: return force_text( model._meta.pk.to_python(value), strings_only=True) else: def m2m_convert(v): return force_text(model._meta.pk.to_python(v), strings_only=True) try: m2m_data[field.name] = [] for pk in field_value: m2m_data[field.name].append(m2m_convert(pk)) except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), pk) # Handle FK fields elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): model = field.remote_field.model if field_value is not None: try: default_manager = model._default_manager field_name = field.remote_field.field_name if issubclass(model, MDR._concept): value, c = model.objects.get_or_create( uuid=field_value, defaults={ 'name': "no name", 'definition': 'no definition' }) data[field.attname] = value.pk #_meta.get_field(field_name).to_python(field_value) elif hasattr(model, 'uuid'): try: value = model.objects.get(uuid=field_value) except ObjectDoesNotExist: # Need to raise error here pass data[field.attname] = value.pk elif hasattr(default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__') and not isinstance( field_value, six.text_type): obj = default_manager.db_manager( db).get_by_natural_key(*field_value) value = getattr( obj, field.remote_field.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if model._meta.pk.remote_field: value = value.pk else: value = model._meta.get_field( field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = model._meta.get_field( field_name).to_python(field_value) except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), field_value) else: data[field.attname] = None # Handle all other fields else: try: data[field.name] = field.to_python(field_value) except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), field_value) with reversion.create_revision(): obj = build_instance(Model, data, db) obj.save() # Get it in the database. for (field_name, field_value) in six.iteritems(d["fields"]): weak_entities = dict( getattr(Model, 'serialize_weak_entities', {})) if field_name in weak_entities.keys(): rel = getattr(Model, weak_entities.get(field_name)) RelModel = rel.rel.related_model other_side = rel.rel.remote_field.name for weak_entity in field_value: # Boy this would be easier if uuids were primary keys :/ extra = {} # Check if any fields are concepts for sub_field_name, sub_value in weak_entity.items(): sub_field = RelModel._meta.get_field( sub_field_name) if sub_field.remote_field and isinstance( sub_field.remote_field, models.ManyToOneRel): sub_model = sub_field.remote_field.model if issubclass(sub_model, MDR._concept): # We have to hope this becomes consistent sub_obj, c = sub_model.objects.get_or_create( uuid=sub_value, defaults={ 'name': "no name", 'definition': 'no definition' }) weak_entity[sub_field_name] = sub_obj weak_entity.update({other_side: obj}) RelModel.objects.update_or_create(**weak_entity) if 'aristotle_mdr.contrib.slots' in settings.INSTALLED_APPS: from aristotle_mdr.contrib.slots.models import Slot for slot in d.get("slots", []): Slot.objects.get_or_create( **{ 'concept': obj, 'name': slot['name'], 'type': slot.get('type', ''), 'value': slot['value'], }) if 'aristotle_mdr.contrib.identifiers' in settings.INSTALLED_APPS: from aristotle_mdr.contrib.identifiers.models import ScopedIdentifier, Namespace for identifier in d.get("identifiers", []): try: org = Organization.objects.get( uuid=identifier['namespace']['naming_authority']) namespace, c = Namespace.objects.get_or_create( shorthand_prefix=identifier['namespace'] ['shorthand_prefix'], naming_authority=org) ScopedIdentifier.objects.get_or_create( **{ 'concept': obj, 'identifier': identifier['identifier'], 'version': identifier.get('version', ""), 'namespace': namespace }) except Exception as e: logger.warning(e) raise #TODO: Better error logging pass for status in d.get("statuses", []): ra, created = MDR.RegistrationAuthority.objects.get_or_create( uuid=uuid.UUID(status["registration_authority"]), defaults={'name': 'Unknown Registration Authority'}) if created: pass # TODO: Log something useful state = { "changeDetails": status.get("change_details", ""), "until_date": status.get("until_date", None), "registrationAuthority": ra, "state": int(status["state"]), "registrationDate": datetime.datetime.strptime(status["registration_date"], '%Y-%m-%d'), "concept": obj } st, c = MDR.Status.objects.get_or_create(**state) if "links" in d.keys( ) and 'aristotle_mdr.contrib.links' in settings.INSTALLED_APPS: from aristotle_mdr.contrib.links import models as link_models # obj_links = link_models.Link.objects.filter(linkend__concept=obj).all().distinct() link_models.Link.objects.filter( linkend__concept=obj).all().distinct().delete() for data in d.get("links", []): rel = link_models.Relation.objects.get( uuid=data['relation']) link = link_models.Link.objects.create(relation=rel) for ordinal, m in enumerate(data['members']): concept = MDR._concept.objects.get(uuid=m['concept']) role, c = link_models.RelationRole.objects.get_or_create( name=m['link'], defaults={ "relation": rel, "ordinal": 0, }) link_models.LinkEnd.objects.update_or_create( link=link, role=role, concept=concept) yield base.DeserializedObject(obj, m2m_data)
def _handle_object(self, node): """ Convert an <group> node to a DeserializedObject. """ # Look up the model using the model loading mechanism. If this fails, # bail. Model = self._get_model_from_node(node, "resname") # Start building a data dictionary from the object. # If the node is missing the pk set it to None bits = node.getAttribute("resname").split(".") keytype = node.getAttribute("d:keytype") or 'pk' if len(bits) == 3: pk = bits[2] else: pk = None data = {} if keytype == 'pk': data[Model._meta.pk.attname] = Model._meta.pk.to_python(pk) else: try: data[Model._meta.pk. attname] = Model.objects.get_by_natural_key(pk).pk except (Model.DoesNotExist, AttributeError): pass # Also start building a dict of m2m data (this is saved as # {m2m_accessor_attribute : [list_of_related_objects]}) m2m_data = defaultdict(list) # Create a reference for genericForeignKeys, if necessary virtual_fields = dict([(x.name, x) for x in Model._meta.virtual_fields]) # Deseralize each field. for field_node in node.getElementsByTagName("trans-unit"): # If the field is missing the name attribute, bail (are you # sensing a pattern here?) field_name = field_node.getAttribute("resname") if not field_name: raise base.DeserializationError( "<trans-unit> node is missing the 'resname' attribute") # Get the field from the Model. This will raise a # FieldDoesNotExist if, well, the field doesn't exist, which will # be propagated correctly. try: field = Model._meta.get_field(field_name) except: if field_name in virtual_fields: field = virtual_fields[field_name] else: raise # As is usually the case, relation fields get the special treatment. if isinstance(field, GenericForeignKey): data[field.name] = self._handle_gfk_field_node( field_node, field) elif field.rel and isinstance(field.rel, models.ManyToManyRel): # There can be multiple instances since each relation has its own tag m2m_data[field.name].append( self._handle_m2m_field_node(field_node, field)) elif field.rel and isinstance(field.rel, models.ManyToOneRel): data[field.attname] = self._handle_fk_field_node( field_node, field) else: if field_node.getElementsByTagName('None'): value = None else: tag = field_node.getElementsByTagName('target') if len(tag) == 0: tag = field_node.getElementsByTagName('source') if len(tag) != 0: value = field.to_python(getInnerText(tag[0]).strip()) else: value = None data[field.name] = value # Return a DeserializedObject so that the m2m data has a place to live. return base.DeserializedObject(Model(**data), m2m_data)
else: data[field.attname] = None # Handle all other fields else: try: data[field.name] = field.to_python(field_value) except Exception as e: raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value) <<<<<<< HEAD obj = base.build_instance(Model, data, db) ======= obj = base.build_instance(Model, data, using) >>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435 yield base.DeserializedObject(obj, m2m_data) def _get_model(model_identifier): <<<<<<< HEAD """ Helper to look up a model from an "app_label.model_name" string. """ ======= """Look up a model from an "app_label.model_name" string.""" >>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435 try: return apps.get_model(model_identifier) except (LookupError, TypeError): raise base.DeserializationError("Invalid model identifier: '%s'" % model_identifier)
def StatePythonDeserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) ignore = options.pop('ignorenonexistent', False) state = options.get('state') for d in object_list: # Look up the model and starting build a dict of data for it. try: Model = _get_model(d["model"], state) except base.DeserializationError: if ignore: continue else: raise data = {} if 'pk' in d: data[Model._meta.pk.attname] = Model._meta.pk.to_python( d.get("pk", None)) m2m_data = {} if is_django_1_7: field_names = Model._meta.get_all_field_names() else: field_names = {f.name for f in Model._meta.get_fields()} # Handle each field for (field_name, field_value) in six.iteritems(d["fields"]): if ignore and field_name not in field_names: # skip fields no longer on model continue if isinstance(field_value, str): field_value = force_text(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.rel and isinstance(field.rel, models.ManyToManyRel): if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__') and not isinstance( value, six.text_type): return field.rel.to._default_manager.db_manager( db).get_by_natural_key(*value).pk else: return force_text( field.rel.to._meta.pk.to_python(value), strings_only=True) else: m2m_convert = lambda v: force_text( field.rel.to._meta.pk.to_python(v), strings_only=True) m2m_data[field.name] = [m2m_convert(pk) for pk in field_value] # Handle FK fields elif field.rel and isinstance(field.rel, models.ManyToOneRel): if field_value is not None: if hasattr(field.rel.to._default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__') and not isinstance( field_value, six.text_type): obj = field.rel.to._default_manager.db_manager( db).get_by_natural_key(*field_value) value = getattr(obj, field.rel.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if field.rel.to._meta.pk.rel: value = value.pk else: value = field.rel.to._meta.get_field( field.rel.field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = field.rel.to._meta.get_field( field.rel.field_name).to_python(field_value) else: data[field.attname] = None # Handle all other fields else: data[field.name] = field.to_python(field_value) obj = base.build_instance(Model, data, db) yield base.DeserializedObject(obj, m2m_data)
def Deserializer(object_list, **options): """ Deserialize simple Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor """ db = options.pop('using', DEFAULT_DB_ALIAS) ignore = options.pop('ignorenonexistent', False) field_names_cache = {} # Model: <list of field_names> for d in object_list: # Look up the model and starting build a dict of data for it. try: Model = _get_model(d["model"]) except base.DeserializationError: if ignore: continue else: raise data = {} if 'pk' in d: try: data[Model._meta.pk.attname] = Model._meta.pk.to_python(d.get('pk')) except Exception as e: raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), None) m2m_data = {} if Model not in field_names_cache: field_names_cache[Model] = {f.name for f in Model._meta.get_fields()} field_names = field_names_cache[Model] # Handle each field for (field_name, field_value) in d["fields"].items(): if ignore and field_name not in field_names: # skip fields no longer on model continue if isinstance(field_value, str): field_value = force_text( field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True ) field = Model._meta.get_field(field_name) # Handle M2M relations if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): model = field.remote_field.model if hasattr(model._default_manager, 'get_by_natural_key'): def m2m_convert(value): if hasattr(value, '__iter__') and not isinstance(value, str): return model._default_manager.db_manager(db).get_by_natural_key(*value).pk else: return force_text(model._meta.pk.to_python(value), strings_only=True) else: def m2m_convert(v): return force_text(model._meta.pk.to_python(v), strings_only=True) try: m2m_data[field.name] = [] for pk in field_value: m2m_data[field.name].append(m2m_convert(pk)) except Exception as e: raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), pk) # Handle FK fields elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): model = field.remote_field.model if field_value is not None: try: default_manager = model._default_manager field_name = field.remote_field.field_name if hasattr(default_manager, 'get_by_natural_key'): if hasattr(field_value, '__iter__') and not isinstance(field_value, str): obj = default_manager.db_manager(db).get_by_natural_key(*field_value) value = getattr(obj, field.remote_field.field_name) # If this is a natural foreign key to an object that # has a FK/O2O as the foreign key, use the FK value if model._meta.pk.remote_field: value = value.pk else: value = model._meta.get_field(field_name).to_python(field_value) data[field.attname] = value else: data[field.attname] = model._meta.get_field(field_name).to_python(field_value) except Exception as e: raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value) else: data[field.attname] = None # Handle all other fields else: try: data[field.name] = field.to_python(field_value) except Exception as e: raise base.DeserializationError.WithData(e, d['model'], d.get('pk'), field_value) obj = base.build_instance(Model, data, db) yield base.DeserializedObject(obj, m2m_data)
def competition_deserializer(object_list, **options): """ Deserialize complex Python objects back into Django ORM instances. It's expected that you pass the Python objects themselves (instead of a stream or a string) to the constructor Links between objects are patched to newly created instances, or existing records in the database. """ processing.reset(len(object_list)) db = options.pop('using', DEFAULT_DB_ALIAS) field_names_cache = {} # Model: <list of field_names> object_list = deque(object_list) system_info = SystemInfo.get_singleton() existing_number_sets = set() dependencies = defaultdict( list ) # Place to hold objects when waiting for referenced objects to load. old_new = {} existing_license_codes = set(LicenseHolder.objects.all().values_list( 'license_code', flat=True)) existing_tags = set(LicenseHolder.objects.all().values_list('existing_tag', flat=True)) existing_license_holder_category = set() more_recently_updated_license_holders = None ts = transaction_save(old_new) while object_list: d = object_list.popleft() # Look up the model and starting build a dict of data for it. Model = _get_model(d["model"]) ts.flush(Model) data = {} if 'pk' in d: try: data[Model._meta.pk.attname] = Model._meta.pk.to_python( d.get('pk')) except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), None) m2m_data = {} pk_old = int(d['pk']) try: field_names = field_names_cache[Model] except KeyError: field_names = field_names_cache[Model] = { f.name for f in Model._meta.get_fields() } # Handle each field has_dependency = False for (field_name, field_value) in d["fields"].items(): if has_dependency: break if field_name not in field_names: # skip fields no longer on model continue if isinstance(field_value, str): field_value = force_text(field_value, options.get("encoding", settings.DEFAULT_CHARSET), strings_only=True) field = Model._meta.get_field(field_name) # Handle M2M relations if field.remote_field and isinstance(field.remote_field, models.ManyToManyRel): model = field.remote_field.model def m2m_convert(v): return force_text(model._meta.pk.to_python(v), strings_only=True) try: m2m_data[field.name] = [] for pk in field_value: m2m_data[field.name].append(m2m_convert(pk)) key = get_key(model, m2m_data[field.name][-1]) try: m2m_data[field.name][-1] = old_new[key] except KeyError: dependencies[key].append(d) has_dependency = True break except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), pk) # Handle FK fields elif field.remote_field and isinstance(field.remote_field, models.ManyToOneRel): model = field.remote_field.model if field_value is not None: try: default_manager = model._default_manager field_name = field.remote_field.field_name data[field.attname] = model._meta.get_field( field_name).to_python(field_value) try: data[field.attname] = old_new[get_key( model, data[field.attname])] except KeyError: dependencies[get_key( model, data[field.attname])].append(d) has_dependency = True except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), field_value) else: data[field.attname] = None # Handle all other fields else: try: data[field.name] = field.to_python(field_value) except Exception as e: raise base.DeserializationError.WithData( e, d['model'], d.get('pk'), field_value) if not has_dependency: instance, existing_instance = _build_instance( Model, data, db, field_names, existing_license_codes, existing_tags, system_info) if Model == Competition: competition = instance more_recently_updated_license_holders = set( Participant.objects.filter( competition__start_date__gt=competition.start_date). values_list('license_holder__id', flat=True)) existing_license_holder_category = set( tuple(lh_cat) for lh_cat in Participant.objects.filter( competition=competition).values_list( 'license_holder', 'category')) db_object = base.DeserializedObject(instance, m2m_data) if existing_instance: # This is an update as there is an existing instance. # Check whether the existing record is more recent and should not be preserved. if Model == LicenseHolder: if not more_recently_updated_license_holders or instance.id not in more_recently_updated_license_holders: ts.save(Model, db_object, instance, pk_old) elif Model == Waiver: if instance.date_signed > existing_instance.date_signed: ts.save(Model, db_object, instance, pk_old) elif Model == LegalEntity: existing_legal_entity = LegalEntity.objects.get( id=instance.id) if instance.waiver_expiry_date > existing_legal_entity.waiver_expiry_date: ts.save(Model, db_object, instance, pk_old) elif Model == NumberSet: # Keep track of this number set if it is new. A new number set means that we do not have to compute # any incremental update, which is much faster. existing_number_sets.add(existing_instance.id) ts.save(Model, db_object, instance, pk_old) elif Model == NumberSetEntry: if not more_recently_updated_license_holders or instance.id not in more_recently_updated_license_holders: if instance.date_lost: if instance.date_lost != existing_instance.date_lost: existing_instance.number_set.set_lost( instance.license_holder, instance.bib, instance.date_lost) else: if existing_instance.bib != instance.bib: existing_instance.number_set.assign_bib( instance.license_holder, instance.bib) else: ts.save(Model, db_object, instance, pk_old) else: if Model == NumberSetEntry and instance.number_set_id in existing_number_sets: ns = instance.number_set if instance.date_lost: ns.set_lost(instance.bib, instance.license_holder, instance.date_lost) else: ns.assign_bib(instance.license_holder, instance.bib) elif Model == Participant: lh_cat = (instance.license_holder_id, instance.category_id if instance.category else None) if lh_cat in existing_license_holder_category: safe_print( u'****Duplicate Participant LicenseHolder Category Integrity Error. Skipped.' ) lh = instance.license_holder safe_print(u' {},{} {}'.format( lh.last_name, lh.first_name, lh.license_code)) else: existing_license_holder_category.add(lh_cat) ts.save(Model, db_object, instance, pk_old) else: ts.save(Model, db_object, instance, pk_old) key = get_key(Model, pk_old) object_list.extend(dependencies.pop(key, [])) if instance.pk: old_new[key] = instance.pk ts.flush()