def Deserializer(stream_or_string, **options): # noqa """ Deserialize a stream or string of JSON data. """ # Local imports to allow using modified versions of `_get_model` # It could be patched in runtime via `unittest.mock.patch` for example from django.core.serializers.python import Deserializer as PythonDeserializer, _get_model ignore = options.pop("ignorenonexistent", False) if not isinstance(stream_or_string, (bytes, str)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode("utf-8") try: for obj in json.loads(stream_or_string): try: Model = _get_model(obj["model"]) except DeserializationError: if ignore: continue else: raise money_fields = {} fields = {} field_names = {field.name for field in Model._meta.get_fields()} for (field_name, field_value) in obj["fields"].items(): if ignore and field_name not in field_names: # skip fields no longer on model continue field = Model._meta.get_field(field_name) if isinstance( field, LinkedCurrencyMoneyField) and field_value is not None: currency_field_name = get_currency_field_name(field_name, field=field) money_fields[field_name] = Money( field_value, get_currency_from_obj(Model, obj, currency_field_name)) # noqa elif isinstance(field, MoneyField) and field_value is not None: money_fields[field_name] = Money( field_value, obj["fields"][get_currency_field_name(field_name)]) else: fields[field_name] = field_value obj["fields"] = fields for inner_obj in PythonDeserializer([obj], **options): for field, value in money_fields.items(): setattr(inner_obj.object, field, value) yield inner_obj except (GeneratorExit, DeserializationError): raise except Exception as exc: raise DeserializationError.with_traceback(DeserializationError(exc), sys.exc_info()[2])
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ def FeatureToPython(dictobj): properties = dictobj['properties'] model_name = properties.pop('model') # Deserialize concrete fields only (bypass dynamic properties) model = _get_model(model_name) field_names = [f.name for f in model._meta.fields] fields = {} for k, v in properties.iteritems(): if k in field_names: fields[k] = v obj = {"model": model_name, "pk": dictobj['id'], "fields": fields} shape = asShape(dictobj['geometry']) obj['geom'] = shape.wkt return obj if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string try: collection = simplejson.load(stream) objects = map(FeatureToPython, collection['features']) for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except Exception, e: # Map to deserializer error raise DeserializationError(e)
def Deserializer(stream_or_string, **options): """Deserialize a stream or string of JSON data.""" if not isinstance(stream_or_string, (bytes, str)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode() try: objects = json.loads(stream_or_string) yield from PythonDeserializer(objects, **options) except Exception as exc: raise DeserializationError() from exc
def __init__(self, *args, **kwargs): # Initialize the base xliff deserializer super().__init__(*args, **kwargs) # Get XLIFF version and initialize deserializer of correct version for event, node in self.event_stream: if event == "START_ELEMENT" and node.nodeName == "xliff": version = node.getAttribute("version") if not version: raise DeserializationError( "The <xliff>-block does not contain a version attribute." ) if version == "1.2": self.xliff_deserializer = xliff1_serializer.Deserializer( *args, **kwargs) return if version == "2.0": self.xliff_deserializer = xliff2_serializer.Deserializer( *args, **kwargs) return raise DeserializationError( f"This serializer cannot process XLIFF version {version}.") raise DeserializationError( "The data does not contain an <xliff>-block.")
def Deserializer(stream_or_string, **options): """Deserialize a stream or string of YAML data.""" if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode() if isinstance(stream_or_string, str): stream = StringIO(stream_or_string) else: stream = stream_or_string try: yield from PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options) except (GeneratorExit, DeserializationError): raise except Exception as exc: raise DeserializationError() from exc
def Deserializer(stream_or_string, **options): if isinstance(stream_or_string, six.string_types): stream_or_string = six.BytesIO(stream_or_string.encode('utf-8')) try: objects = ijson.items(stream_or_string, 'item') for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
def Deserializer(stream_or_string, **options): """Deserialize a stream or string of JSON data.""" if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode() if isinstance(stream_or_string, (bytes, str)): stream_or_string = stream_or_string.split("\n") for line in stream_or_string: if not line.strip(): continue try: yield from PythonDeserializer([json.loads(line)], **options) except (GeneratorExit, DeserializationError): raise except Exception as exc: raise DeserializationError() from exc
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of YAML data. """ if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string try: for obj in PythonDeserializer(yaml.safe_load(stream), **options): yield obj except GeneratorExit: raise except Exception, e: # Map to deserializer error raise DeserializationError(e)
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if isinstance(stream_or_string, basestring): stream = BytesIO(stream_or_string) else: stream = stream_or_string try: for obj in PythonDeserializer(json.load(stream), **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error raise DeserializationError(e)
def to_model_instance(self, to, check_hostname=None): if self.is_consumed: return True self.is_success = None self.destination = to self.check_hostname = check_hostname for obj in self.deserialized_objects(): if self.ignore_obj(obj): pass elif self.action == 'I' or self.action == 'U': self.save_obj(obj) elif self.action == 'D': self.delete_obj(obj) else: raise DeserializationError( 'Unknown object action. Expected one of [I, U, D]. Got {}'.format(self.action)) return self.is_consumed or self.is_ignored
def Deserializer(stream_or_string, **options): """Deserialize a stream or string of JSON data. This is a slightly modified copy of Django implementation with additional argument <object_hook> in json.loads""" if not isinstance(stream_or_string, (bytes, str)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode() try: objects = json.loads(stream_or_string, object_hook=object_hook) yield from PythonDeserializer(objects, **options) except Exception as exc: # ugly construction to overcome pylint's warning # "The except handler raises immediately" if isinstance(exc, (GeneratorExit, DeserializationError)): raise raise DeserializationError() from exc
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if not isinstance(stream_or_string, (bytes, six.string_types)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf-8') try: objects = json.loads(stream_or_string) for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of YAML data. """ if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf-8') if isinstance(stream_or_string, six.string_types): stream = StringIO(stream_or_string) else: stream = stream_or_string try: for obj in PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error raise DeserializationError(e)
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf-8') try: if isinstance(stream_or_string, basestring): objects = json.loads(stream_or_string) else: objects = json.load(stream_or_string) for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error raise DeserializationError(e)
def Deserializer(stream_or_string, **options): # noqa """ Deserialize a stream or string of JSON data. """ ignore = options.pop('ignorenonexistent', False) if not isinstance(stream_or_string, (bytes, six.string_types)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf-8') try: for obj in json.loads(stream_or_string): try: Model = _get_model(obj['model']) except DeserializationError: if ignore: continue else: raise money_fields = {} fields = {} field_names = {field.name for field in Model._meta.get_fields()} for (field_name, field_value) in six.iteritems(obj['fields']): if ignore and field_name not in field_names: # skip fields no longer on model continue field = Model._meta.get_field(field_name) if isinstance(field, MoneyField) and field_value is not None: money_fields[field_name] = Money( field_value, obj['fields'][get_currency_field_name(field_name)]) else: fields[field_name] = field_value obj['fields'] = fields for inner_obj in PythonDeserializer([obj], **options): for field, value in money_fields.items(): setattr(inner_obj.object, field, value) yield inner_obj except (GeneratorExit, DeserializationError): raise except Exception as exc: six.reraise(DeserializationError, DeserializationError(exc), sys.exc_info()[2])
def Deserializer(stream, **options): """ Deserialize a stream of JSON data using iterative ijson so we may not load the whole string into memory. """ if isinstance(stream, (bytes, six.string_types)): raise TypeError( 'Use iloaddata/ijson with streams only. For strings use plain loaddata/json.loads' ) try: objects = ijson.items(stream, 'item') for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ geometry_field = options.get("geometry_field", "geom") def FeatureToPython(dictobj): properties = dictobj['properties'] model_name = options.get("model_name") or properties.pop('model') # Deserialize concrete fields only (bypass dynamic properties) model = _get_model(model_name) field_names = [f.name for f in model._meta.fields] fields = {} for k, v in iteritems(properties): if k in field_names: fields[k] = v obj = { "model": model_name, "pk": dictobj.get('id') or properties.get('id'), "fields": fields } if isinstance(model._meta.get_field(geometry_field), GeoJSONField): obj['fields'][geometry_field] = dictobj['geometry'] else: shape = GEOSGeometry(json.dumps(dictobj['geometry'])) obj['fields'][geometry_field] = shape.wkt return obj if isinstance(stream_or_string, string_types): stream = StringIO(stream_or_string) else: stream = stream_or_string try: collection = json.load(stream) objects = [FeatureToPython(f) for f in collection['features']] for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error raise DeserializationError(repr(e))
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. Note: no change from Django version, but needed to import here to use the versioned python deserializer (see the import at the top for PythonDeserializer) """ if isinstance(stream_or_string, basestring): stream = StringIO(stream_or_string) else: stream = stream_or_string try: for obj in PythonDeserializer(simplejson.load(stream), **options): yield obj except GeneratorExit: raise except Exception, e: # Map to deserializer error raise DeserializationError(e)
def FeatureToPython(dictobj): properties = dictobj['properties'] model_name = options.get("model_name") or properties.pop('model') # Deserialize concrete fields only (bypass dynamic properties) model = _get_model(model_name) field_names = [f.name for f in model._meta.fields] fields = {} for k, v in iteritems(properties): if k in field_names: fields[k] = v obj = { "model": model_name, "pk": dictobj.get('id') or properties.get('id'), "fields": fields } if asShape is None: raise DeserializationError('shapely is not installed') shape = asShape(dictobj['geometry']) obj['fields'][geometry_field] = shape.wkt return obj
def test_classify_callback_handles_exceptions_when_message_is_not_json( self): # mock the deserialize method and force it to raise an exception classify.serializers.deserialize = mock.MagicMock( side_effect=DeserializationError('foo')) # regenerate the command class being tested as we mocked an extra module above self.command = classify.Command() # make it look like there is a trained classifier self.command.classifier = mock.MagicMock() # the supposed body of the enqueued item body = '' # call the method being tested self.command.classify_callback(self.channel, mock.MagicMock(), mock.MagicMock(), body) # un-acknowledged the item got from the queue self.channel.basic_nack.assert_called_once() # error was logged self.logger.error.assert_called_once_with( 'Classifier failed to deserialize body.')
def require_attribute(node, attribute): """ Get the attribute of a node and throw an error if it evaluates to ``False`` :param node: The current xml node of the object :type node: xml.dom.minidom.Element :param attribute: The name of the requested attribute :type attribute: str :raises ~django.core.serializers.base.DeserializationError: If the deserialization fails :return: The value name of the requested attribute :rtype: str """ value = node.getAttribute(attribute) if not value: raise DeserializationError( f"<{node.nodeName}> node is missing the {attribute} attribute" ) return value
def _handle_object(self, node): ret = {} if node.hasAttribute("pk"): ret['pk'] = node.getAttribute('pk') else: ret['pk'] = None ret['model'] = node.getAttribute('model') fields = {} for field_node in node.getElementsByTagName("field"): # If the field is missing the name attribute, bail name = field_node.getAttribute("name") rel = field_node.getAttribute("rel") if not name: raise DeserializationError( "<field> node is missing the 'name' attribute") if field_node.getElementsByTagName('None'): value = None elif rel == 'ManyToManyRel': value = [ n.getAttribute('pk') for n in field_node.getElementsByTagName('object') ] elif field_node.getElementsByTagName('natural'): value = [ getInnerText(n).strip() for n in field_node.getElementsByTagName('natural') ] else: value = getInnerText(field_node).strip() fields[name] = value ret['fields'] = fields return ret
def Deserializer(stream_or_string, **options): """ Deserialize a stream or string of JSON data. """ if not isinstance(stream_or_string, (bytes, six.string_types)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf8') try: objects = json.loads(stream_or_string) for obj in PythonDeserializer(objects, **options): yield obj except GeneratorExit: raise except AttributeError as e: # if the fixture data doesn't match the given class, it's probably # an overridden class, so we should ignore the errors so a future # fixture can be used. loaddata will ignore AttributeError but # fail on DeserializationError raise except Exception as e: six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2])
def deserialize_collection(self, collection_path): log.info("Deserializing collection at %s", collection_path) # Determine paths and make sure they exist before deserializing records_path = os.path.join(collection_path, RECORD_DIR) self._check_exists(records_path) group_record_filepath = os.path.join(records_path, GROUP_FILENAME) self._check_exists(group_record_filepath) collection_set_record_filepath = os.path.join(records_path, COLLECTION_SET_FILENAME) self._check_exists(collection_set_record_filepath) historical_collection_set_record_filepath = os.path.join( records_path, HISTORICAL_COLLECTION_SET_FILENAME) self._check_exists(historical_collection_set_record_filepath) collection_record_filepath = os.path.join(records_path, COLLECTION_FILENAME) self._check_exists(collection_record_filepath) historical_collection_record_filepath = os.path.join( records_path, HISTORICAL_COLLECTION_FILENAME) self._check_exists(historical_collection_record_filepath) user_record_filepath = os.path.join(records_path, USER_FILENAME) self._check_exists(user_record_filepath) credential_record_filepath = os.path.join(records_path, CREDENTIAL_FILENAME) self._check_exists(credential_record_filepath) historical_credential_record_filepath = os.path.join( records_path, HISTORICAL_CREDENTIAL_FILENAME) self._check_exists(historical_credential_record_filepath) seed_filepath = os.path.join(records_path, SEED_FILENAME) self._check_exists(seed_filepath) historical_seed_filepath = os.path.join(records_path, HISTORICAL_SEED_FILENAME) self._check_exists(historical_seed_filepath) harvest_filepath = os.path.join(records_path, HARVEST_FILENAME) self._check_exists(harvest_filepath) harvest_stats_filepath = os.path.join(records_path, HARVEST_STATS_FILENAME) self._check_exists(harvest_stats_filepath) warcs_filepath = os.path.join(records_path, WARC_FILENAME) self._check_exists(warcs_filepath) # Only proceed with deserialization if collection doesn't already exist collection_id = self._load_record( collection_record_filepath)[0]["fields"]["collection_id"] if not Collection.objects.filter(collection_id=collection_id).exists(): log.debug( "Collection does not exist, so proceeding with deserialization" ) collection_set_id = self._load_record( collection_set_record_filepath )[0]["fields"]["collection_set_id"] # Only proceed with deserialization of collection set if it doesn't already exist # Make sure that in collection path is correct expected_collection_path = get_collection_path_by_id( collection_set_id=collection_set_id, collection_id=collection_id, sfm_data_dir=self.data_dir) if not os.path.exists( expected_collection_path) or not os.path.samefile( expected_collection_path, collection_path): raise DeserializationError( "Collection path is {}, but should be {}".format( collection_path, expected_collection_path)) # Groups first self._deserialize_groups(group_record_filepath) # Users self._deserialize_users(user_record_filepath) if not CollectionSet.objects.filter( collection_set_id=collection_set_id).exists(): log.debug( "Collection set does not exist, so proceeding with deserialization" ) # Collection set self._deserialize(collection_set_record_filepath) # Historical collection set self._deserialize_historical_objects( historical_collection_set_record_filepath, collection_set_record_filepath, "collection_set_id", CollectionSet) # Add a history note collection_set = CollectionSet.objects.get( collection_set_id=collection_set_id) collection_set.history_note = "Collection set imported." collection_set.save(force_history=True) else: log.warning( "Collection set already exists, so not deserializing") # Credentials and credential history self._deserialize_credentials( credential_record_filepath, historical_credential_record_filepath) # Collection self._deserialize(collection_record_filepath) # Collection history self._deserialize_historical_objects( historical_collection_record_filepath, collection_record_filepath, "collection_id", Collection) # Seeds self._deserialize(seed_filepath) # Historical seeds self._deserialize_historical_objects(historical_seed_filepath, seed_filepath, "seed_id", Seed) # Harvests self._deserialize(harvest_filepath) # Harvest stats self._deserialize(harvest_stats_filepath) # Warcs self._deserialize(warcs_filepath) # Turn off collection collection = Collection.objects.get_by_natural_key(collection_id) if collection.is_on: log.debug("Turning off collection") collection.is_on = False collection.history_note = "Turned off as part of import of this collection." collection.save() # Add history note for import collection.history_note = "Collection imported." collection.save(force_history=True) else: log.warning("Collection already exists, so not deserializing")
def Deserializer(stream_or_string, **options): # noqa """ Deserialize a stream or string of JSON data. """ # Copied almost without changes from djmoney.serializers (django-money). # Adding support for situation where old models to be deserialized have # price field, but not price_currency field. # In Ralph, price field existed before in various models as # a Decimal field. All price fields were migrated to MoneyField # without changing the original field name. This can cause problems # in original django-money's implementation of Deserializer. # This updated Deserializer is needed to get reversion (django-reversion) # to work in circumstances described above. from django.core.serializers.python import \ Deserializer as PythonDeserializer, _get_model ignore = options.pop("ignorenonexistent", False) if not isinstance(stream_or_string, (bytes, six.string_types)): stream_or_string = stream_or_string.read() if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode("utf-8") try: for obj in json.loads(stream_or_string): try: Model = _get_model(obj["model"]) except DeserializationError: if ignore: continue else: raise money_fields = {} fields = {} field_names = {field.name for field in Model._meta.get_fields()} for (field_name, field_value) in six.iteritems(obj["fields"]): if ignore and field_name not in field_names: # skip fields no longer on model continue field = Model._meta.get_field(field_name) if isinstance(field, MoneyField) and field_value is not None: try: currency = \ obj["fields"][get_currency_field_name(field_name)] except KeyError: currency = DEFAULT_CURRENCY_CODE money_fields[field_name] = Money(field_value, currency) else: fields[field_name] = field_value obj["fields"] = fields for inner_obj in PythonDeserializer([obj], **options): for field, value in money_fields.items(): setattr(inner_obj.object, field, value) yield inner_obj except (GeneratorExit, DeserializationError): raise except Exception as exc: six.reraise( DeserializationError, DeserializationError(exc), sys.exc_info()[2] )
def handle_object(self, node): """ Convert a ``<file>``-node to a ``DeserializedObject``. :param node: The current xml node of the object :type node: xml.dom.minidom.Element :raises ~django.core.serializers.base.DeserializationError: If the deserialization fails :raises ~django.core.exceptions.FieldDoesNotExist: If the XLIFF file contains a field which doesn't exist on the :return: The deserialized page translation :rtype: django.core.serializers.base.DeserializedObject PageTranslation model """ # Get page translation (process varies for the different xliff versions) page_translation = self.get_object(node) logger.debug( "Existing page translation: %r", page_translation, ) # Increment the version number page_translation.version = page_translation.version + 1 # Make sure object is not in translation anymore if it was before page_translation.currently_in_translation = False # Set the id to None to make sure a new object is stored in the database when save() is called page_translation.id = None # Deserialize each field. for field_node in node.getElementsByTagName(self.unit_node): # Check to which attribute this resource belongs to field_name = self.require_attribute(field_node, "resname") # Get the field from the PageTranslation model try: field = page_translation._meta.get_field(field_name) except FieldDoesNotExist as e: # If the field doesn't exist, check if a legacy field is supported field_name = settings.XLIFF_LEGACY_FIELDS.get(field_name) try: field = page_translation._meta.get_field(field_name) except FieldDoesNotExist: # If the legacy field doesn't exist as well, just raise the initial exception # pylint: disable=raise-missing-from raise e # Now get the actual target value of the field target = field_node.getElementsByTagName("target") if not target: raise DeserializationError( f"Field {field_name} does not contain a <target> node." ) # Set the field attribute of the page translation to the new target value setattr( page_translation, field_name, field.to_python(xml_serializer.getInnerText(target[0]).strip()), ) logger.debug("Deserialized page translation: %r", page_translation) # Return a DeserializedObject return DeserializedObject(page_translation)
Deserialize a stream or string of YAML data. """ if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf-8') if isinstance(stream_or_string, six.string_types): ======= """Deserialize a stream or string of YAML data.""" if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode() if isinstance(stream_or_string, str): >>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435 stream = StringIO(stream_or_string) else: stream = stream_or_string try: <<<<<<< HEAD for obj in PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options): yield obj except GeneratorExit: raise except Exception as e: # Map to deserializer error six.reraise(DeserializationError, DeserializationError(e), sys.exc_info()[2]) ======= yield from PythonDeserializer(yaml.load(stream, Loader=SafeLoader), **options) except (GeneratorExit, DeserializationError): raise except Exception as exc: raise DeserializationError() from exc >>>>>>> 37c99181c9a6b95433d60f8c8ef9af5731096435
def parse(self, stream): try: return json.load(stream) except Exception as e: # Map to deserializer error raise DeserializationError(e)