def _type_single(self, value, _type):
' apply type to the single value '
if value is None or _type in (None, NoneType):
# don't convert null values
# default type is the original type if none set
pass
elif isinstance(value, _type): # or values already of correct type
# normalize all dates to epochs
value = dt2ts(value) if _type in [datetime, date] else value
else:
if _type in (datetime, date):
# normalize all dates to epochs
value = dt2ts(value)
elif _type in (unicode, str):
# make sure all string types are properly unicoded
value = to_encoding(value)
else:
try:
value = _type(value)
except Exception:
value = to_encoding(value)
logger.error("typecast failed: %s(value=%s)" % (
_type.__name__, value))
raise
return value
def _type_single(self, value, _type):
' apply type to the single value '
if value is None or _type in (None, NoneType):
# don't convert null values
# default type is the original type if none set
pass
elif isinstance(value, _type): # or values already of correct type
# normalize all dates to epochs
value = dt2ts(value) if _type in [datetime, date] else value
else:
if _type in (datetime, date):
# normalize all dates to epochs
value = dt2ts(value)
elif _type in (unicode, str):
# make sure all string types are properly unicoded
value = to_encoding(value)
else:
try:
value = _type(value)
except Exception:
value = to_encoding(value)
logger.error("typecast failed: %s(value=%s)" %
(_type.__name__, value))
raise
return value
def _normalize_key(self, key):
key = to_encoding(key).lower()
if key != '__v__':
# skip internal metrique version field
key = self.SPACE_RE.sub('_', key)
key = self.FIELDS_RE.sub('', key)
key = self.UNDA_RE.sub('_', key)
return key
def _normalize_key(self, key):
key = to_encoding(key).lower()
if key != '__v__':
# skip internal metrique version field
key = self.SPACE_RE.sub('_', key)
key = self.FIELDS_RE.sub('', key)
key = self.UNDA_RE.sub('_', key)
return key
def gen_id(_oid, _start, _end=None):
# if the object is 'current value' without _end,
# use just str of _oid (DEFAULT)
if _oid is None:
raise ValueError("_oid must be defined!")
elif _end:
_oid = '%s:%s' % (_oid, _start)
else:
_oid = to_encoding(_oid)
return _oid
def gen_id(_oid, _start, _end=None):
# if the object is 'current value' without _end,
# use just str of _oid (DEFAULT)
if _oid is None:
raise ValueError("_oid must be defined!")
elif _end:
_oid = '%s:%s' % (_oid, _start)
else:
_oid = to_encoding(_oid)
return _oid
def __getitem__(self, key):
'''
Support getting index slices or individiual objects. Expected
_id index values.
'''
if isinstance(key, slice):
keys = sorted(self.store.keys())[key]
return [dict(self.store[i]) for i in keys]
else:
key = to_encoding(key)
return dict(self.store[key])
def __getitem__(self, key):
'''
Support getting index slices or individiual objects. Expected
_id index values.
'''
if isinstance(key, slice):
keys = sorted(self.store.keys())[key]
return [dict(self.store[i]) for i in keys]
else:
key = to_encoding(key)
return dict(self.store[key])
def _unwrap(self, value):
if type(value) is buffer:
# unwrap/convert the aggregated string 'buffer'
# objects to string
value = to_encoding(value)
# FIXME: this might cause issues if the buffered
# text has " quotes...
value = value.replace('"', '').strip()
if not value:
value = None
else:
value = value.split('\n')
return value
def setup(args, cmd, pip=False):
pre = not getattr(args, 'no_pre', False)
if pip and not pre:
cmd += ' --pre'
if isinstance(cmd, basestring):
cmd = cmd.strip()
else:
cmd = ' '.join([s.strip() for s in cmd])
if pip:
out = utils.sys_call('pip %s -e .' % cmd)
else:
out = utils.sys_call('python setup.py %s' % cmd)
logger.info(utils.to_encoding(out))
def setup(args, cmd, pip=False):
pre = not getattr(args, 'no_pre', False)
if pip and not pre:
cmd += ' --pre'
if isinstance(cmd, basestring):
cmd = cmd.strip()
else:
cmd = ' '.join([s.strip() for s in cmd])
if pip:
out = utils.sys_call('pip %s -e .' % cmd)
else:
out = utils.sys_call('python setup.py %s' % cmd)
logger.info(utils.to_encoding(out))
def _unwrap(self, value):
if type(value) is buffer:
# unwrap/convert the aggregated string 'buffer'
# objects to string
value = to_encoding(value)
# FIXME: this might cause issues if the buffered
# text has " quotes...
value = value.replace('"', '').strip()
if not value:
value = None
else:
value = value.split('\n')
return value
def _prep_object(self, obj):
obj = self._normalize_keys(obj)
schema = self.schema
if not schema:
# build schema off normalized key version
_obj = self._normalize_keys(obj)
# cache a key map from old key->normalized keys
self._key_map = {k: self._normalize_key(k) for k in obj.iterkeys()}
# in the case we don't have a schema already defined, we need to
# build on now; all objects are assumed to have the SAME SCHEMA!
schema = autoschema([_obj], exclude_keys=self.RESTRICTED_KEYS)
self.config['schema'] = schema
# optimization; lookup in local scope
kmap = self._key_map or {}
for key, value in obj.items():
# map original key to normalized key, if normal map exists
_key = kmap.get(key) or key
_schema = schema.get(_key) or {}
try:
value = self._prep_value(value, schema=_schema)
except Exception as e:
# make sure the value contents are loggable
_value = to_encoding(value)
obj.setdefault('_e', {})
msg = 'prep(key=%s, value=%s) failed: %s' % (_key, _value, e)
logger.error(msg)
# set error field with original values
obj['_e'].update({_key: value})
# FIXME: should we leave original as-is? if not of correct
# type, etc, this might cause problems
# normalize invalid value to None
value = None
obj[key] = value
variants = self._add_variants(_key, value, _schema)
obj.update(variants)
obj['_v'] = self.version
obj = self._object_cls(**obj)
return obj
def _prep_object(self, obj):
obj = self._normalize_keys(obj)
schema = self.schema
if not schema:
# build schema off normalized key version
_obj = self._normalize_keys(obj)
# cache a key map from old key->normalized keys
self._key_map = {k: self._normalize_key(k) for k in obj.iterkeys()}
# in the case we don't have a schema already defined, we need to
# build on now; all objects are assumed to have the SAME SCHEMA!
schema = autoschema([_obj], exclude_keys=self.RESTRICTED_KEYS)
self.config['schema'] = schema
# optimization; lookup in local scope
kmap = self._key_map or {}
for key, value in obj.items():
# map original key to normalized key, if normal map exists
_key = kmap.get(key) or key
_schema = schema.get(_key) or {}
try:
value = self._prep_value(value, schema=_schema)
except Exception as e:
# make sure the value contents are loggable
_value = to_encoding(value)
obj.setdefault('_e', {})
msg = 'prep(key=%s, value=%s) failed: %s' % (_key, _value, e)
logger.error(msg)
# set error field with original values
obj['_e'].update({_key: value})
# FIXME: should we leave original as-is? if not of correct
# type, etc, this might cause problems
# normalize invalid value to None
value = None
obj[key] = value
variants = self._add_variants(_key, value, _schema)
obj.update(variants)
obj['_v'] = self.version
obj = self._object_cls(**obj)
return obj
def test_to_encoding():
from metrique.utils import to_encoding
str_utf8 = unicode('--台北--')
str_ = str('hello')
assert to_encoding(str_utf8, 'utf-8')
assert to_encoding(str_, 'utf-8')
assert to_encoding(str_utf8, 'ascii')
assert to_encoding(str_, 'ascii')
try:
to_encoding(str_utf8, 'ascii', errors='strict')
except UnicodeEncodeError:
pass
else:
assert False, "Manged to convert unicode to ascii when using strict"
def test_to_encoding():
from metrique.utils import to_encoding
str_utf8 = unicode('--台北--')
str_ = str('hello')
assert to_encoding(str_utf8, 'utf-8')
assert to_encoding(str_, 'utf-8')
assert to_encoding(str_utf8, 'ascii')
assert to_encoding(str_, 'ascii')
try:
to_encoding(str_utf8, 'ascii', errors='strict')
except UnicodeEncodeError:
pass
else:
assert False, "Manged to convert unicode to ascii when using strict"
def process_bind_param(self, value, dialect):
if not (value is None or isinstance(value, unicode)):
value = to_encoding(value)
return value
def process_bind_param(self, value, dialect):
return None if value is None else to_encoding(
json.dumps(
value, default=json_encode_default, ensure_ascii=False))
def pop(self, key):
key = to_encoding(key)
return self.store.pop(key)
def process_bind_param(self, value, dialect):
return None if value is None else to_encoding(
json.dumps(value, default=json_encode_default,
ensure_ascii=False))
def process_bind_param(self, value, dialect):
if not (value is None or isinstance(value, unicode)):
value = to_encoding(value)
return value
def __contains__(self, key):
key = to_encoding(key)
return key in self.store.keys()
def __delitem__(self, key):
key = to_encoding(key)
del self.store[key]
def __delitem__(self, key):
key = to_encoding(key)
del self.store[key]
def pop(self, key):
key = to_encoding(key)
return self.store.pop(key)
def __contains__(self, key):
key = to_encoding(key)
return key in self.store.keys()
