def stringify(
value: Any, encoding_default: str = DEFAULT_ENCODING, encoding: Optional[str] = None
) -> Optional[str]:
"""Brute-force convert a given object to a string.
This will attempt an increasingly mean set of conversions to make a given
object into a unicode string. It is guaranteed to either return unicode or
None, if all conversions failed (or the value is indeed empty).
"""
if value is None:
return None
if isinstance(value, str):
return _clean_empty(value)
if isinstance(value, (date, datetime)):
return value.isoformat()
elif isinstance(value, (float, Decimal)):
return Decimal(value).to_eng_string()
elif isinstance(value, bytes):
if encoding is None:
encoding = guess_encoding(value, default=encoding_default)
value = value.decode(encoding, "replace")
value = remove_unsafe_chars(value)
if value is None:
return None
return _clean_empty(value)
return _clean_empty(str(value))
def stringify(value, encoding_default='utf-8', encoding=None):
"""Brute-force convert a given object to a string.
This will attempt an increasingly mean set of conversions to make a given
object into a unicode string. It is guaranteed to either return unicode or
None, if all conversions failed (or the value is indeed empty).
"""
if value is None:
return None
if not isinstance(value, six.text_type):
if isinstance(value, (date, datetime)):
return value.isoformat()
elif isinstance(value, (float, Decimal)):
return Decimal(value).to_eng_string()
elif isinstance(value, six.binary_type):
if encoding is None:
encoding = guess_encoding(value, default=encoding_default)
value = value.decode(encoding, 'replace')
value = remove_byte_order_mark(value)
value = remove_unsafe_chars(value)
else:
value = six.text_type(value)
# XXX: is this really a good idea?
value = value.strip()
if not len(value):
return None
return value
def _convert_value(self, value, table, column):
if isinstance(column.type, (types.DateTime, types.Date)):
if value in ('0000-00-00 00:00:00', '0000-00-00'):
value = None
if isinstance(column.type, (types.String, types.Unicode)):
if isinstance(value, str):
value = remove_unsafe_chars(value)
return value
def sanitize_text(text, encoding=DEFAULT_ENCODING):
text = stringify(text, encoding_default=encoding)
text = remove_unsafe_chars(text)
if text is None:
return
text = text.encode(encoding, 'replace')
text = text.decode(encoding, 'strict')
return text
def safe_string(data, encoding_default='utf-8', encoding=None):
"""Stringify and round-trip through encoding."""
data = stringify(data,
encoding_default=encoding_default,
encoding=encoding)
data = remove_unsafe_chars(data)
if data is None:
return
data = data.encode(encoding_default, 'replace')
data = data.decode(encoding_default, 'strict')
return data
def sanitize_text(text, encoding=DEFAULT_ENCODING):
text = stringify(text, encoding_default=encoding)
if text is not None:
try:
text = compose_nfc(text)
except (SystemError, Exception) as ex:
log.warning("Cannot NFC text: %s", ex)
return None
text = remove_unsafe_chars(text)
text = text.encode(DEFAULT_ENCODING, "replace")
return text.decode(DEFAULT_ENCODING, "replace")
def sanitize_text(text: Any, encoding: str = DEFAULT_ENCODING) -> Optional[str]:
text = stringify(text, encoding_default=encoding)
if text is None:
return None
try:
text = compose_nfc(text)
except (SystemError, Exception) as ex:
log.warning("Cannot NFC text: %s", ex)
return None
text = remove_unsafe_chars(text)
if text is None:
return None
byte_text = text.encode(DEFAULT_ENCODING, "replace")
return cast(str, byte_text.decode(DEFAULT_ENCODING, "replace"))
def _getStringStream(self, filename):
"""Gets a string representation of the requested filename.
Checks for both ASCII and Unicode representations and returns
a value if possible. If there are both ASCII and Unicode
versions, then the parameter /prefer/ specifies which will be
returned.
"""
if isinstance(filename, list):
# Join with slashes to make it easier to append the type
filename = "/".join(filename)
value = windowsUnicode(self._getStream(filename + '001F'))
if value is None:
raw = self._getStream(filename + '001E')
try:
value = decode_utf7(raw)
except Exception:
encoding = guess_encoding(raw)
value = raw.decode(encoding, 'replace')
if value is not None and len(value):
return remove_unsafe_chars(value)