def _set_attr(obj_, path, value):
obj = _get_attr(obj_, path[:-1])
if obj is None: # DELIBERATE USE OF `is`: WE DO NOT WHAT TO CATCH Null HERE (THEY CAN BE SET)
obj = _get_attr(obj_, path[:-1])
if obj is None:
get_logger().error(PATH_NOT_FOUND+" tried to get attribute of None")
attr_name = path[-1]
# ACTUAL SETTING OF VALUE
try:
old_value = _get_attr(obj, [attr_name])
if old_value == None:
old_value = None
new_value = value
elif value == None:
new_value = None
else:
new_value = old_value.__class__(value) # TRY TO MAKE INSTANCE OF SAME CLASS
except Exception as e:
old_value = None
new_value = value
try:
setattr(obj, attr_name, new_value)
return old_value
except Exception as e:
try:
obj[attr_name] = new_value
return old_value
except Exception as f:
get_logger().error(PATH_NOT_FOUND, cause=e)
def _set_attr(obj_, path, value):
obj = _get_attr(obj_, path[:-1])
if obj is None: # DELIBERATE USE OF `is`: WE DO NOT WHAT TO CATCH Null HERE (THEY CAN BE SET)
obj = _get_attr(obj_, path[:-1])
if obj is None:
get_logger().error(PATH_NOT_FOUND +
" tried to get attribute of None")
attr_name = path[-1]
# ACTUAL SETTING OF VALUE
try:
old_value = _get_attr(obj, [attr_name])
if old_value == None:
old_value = None
new_value = value
elif value == None:
new_value = None
else:
new_value = old_value.__class__(
value) # TRY TO MAKE INSTANCE OF SAME CLASS
except Exception as e:
old_value = None
new_value = value
try:
setattr(obj, attr_name, new_value)
return old_value
except Exception as e:
try:
obj[attr_name] = new_value
return old_value
except Exception as f:
get_logger().error(PATH_NOT_FOUND, cause=e)
def _get_attr(obj, path):
if not path:
return obj
attr_name = path[0]
if isinstance(obj, ModuleType):
if attr_name in obj.__dict__:
return _get_attr(obj.__dict__[attr_name], path[1:])
elif attr_name in dir(obj):
return _get_attr(obj[attr_name], path[1:])
# TRY FILESYSTEM
File = get_module("mo_files").File
possible_error = None
python_file = (File(obj.__file__).parent / attr_name).set_extension("py")
python_module = (File(obj.__file__).parent / attr_name / "__init__.py")
if python_file.exists or python_module.exists:
try:
# THIS CASE IS WHEN THE __init__.py DOES NOT IMPORT THE SUBDIR FILE
# WE CAN STILL PUT THE PATH TO THE FILE IN THE from CLAUSE
if len(path) == 1:
# GET MODULE OBJECT
output = __import__(obj.__name__ + str(".") + str(attr_name), globals(), locals(), [str(attr_name)], 0)
return output
else:
# GET VARIABLE IN MODULE
output = __import__(obj.__name__ + str(".") + str(attr_name), globals(), locals(), [str(path[1])], 0)
return _get_attr(output, path[1:])
except Exception as e:
Except = get_module("mo_logs.exceptions.Except")
possible_error = Except.wrap(e)
# TRY A CASE-INSENSITIVE MATCH
matched_attr_name = lower_match(attr_name, dir(obj))
if not matched_attr_name:
get_logger().warning(PATH_NOT_FOUND + "({{name|quote}}) Returning None.", name=attr_name, cause=possible_error)
elif len(matched_attr_name) > 1:
get_logger().error(AMBIGUOUS_PATH_FOUND + " {{paths}}", paths=attr_name)
else:
return _get_attr(obj[matched_attr_name[0]], path[1:])
try:
obj = obj[int(attr_name)]
return _get_attr(obj, path[1:])
except Exception:
pass
try:
obj = getattr(obj, attr_name)
return _get_attr(obj, path[1:])
except Exception:
pass
try:
obj = obj[attr_name]
return _get_attr(obj, path[1:])
except Exception as f:
return None
def _get_attr(obj, path):
if not path:
return obj
attr_name = path[0]
if isinstance(obj, ModuleType):
if attr_name in obj.__dict__:
return _get_attr(obj.__dict__[attr_name], path[1:])
elif attr_name in dir(obj):
return _get_attr(obj[attr_name], path[1:])
# TRY FILESYSTEM
File = get_module("mo_files").File
possible_error = None
python_file = (File(obj.__file__).parent / attr_name).set_extension("py")
python_module = (File(obj.__file__).parent / attr_name / "__init__.py")
if python_file.exists or python_module.exists:
try:
# THIS CASE IS WHEN THE __init__.py DOES NOT IMPORT THE SUBDIR FILE
# WE CAN STILL PUT THE PATH TO THE FILE IN THE from CLAUSE
if len(path) == 1:
# GET MODULE OBJECT
output = __import__(obj.__name__ + str(".") + str(attr_name), globals(), locals(), [str(attr_name)], 0)
return output
else:
# GET VARIABLE IN MODULE
output = __import__(obj.__name__ + str(".") + str(attr_name), globals(), locals(), [str(path[1])], 0)
return _get_attr(output, path[1:])
except Exception as e:
Except = get_module("mo_logs.exceptions.Except")
possible_error = Except.wrap(e)
# TRY A CASE-INSENSITIVE MATCH
matched_attr_name = lower_match(attr_name, dir(obj))
if not matched_attr_name:
get_logger().warning(PATH_NOT_FOUND + "({{name|quote}}) Returning None.", name=attr_name, cause=possible_error)
elif len(matched_attr_name) > 1:
get_logger().error(AMBIGUOUS_PATH_FOUND + " {{paths}}", paths=attr_name)
else:
return _get_attr(obj[matched_attr_name[0]], path[1:])
try:
obj = obj[int(attr_name)]
return _get_attr(obj, path[1:])
except Exception:
pass
try:
obj = getattr(obj, attr_name)
return _get_attr(obj, path[1:])
except Exception:
pass
try:
obj = obj[attr_name]
return _get_attr(obj, path[1:])
except Exception as f:
return None
def literal_field(field):
"""
RETURN SAME WITH DOTS (`.`) ESCAPED
"""
try:
return field.replace(".", "\\.")
except Exception as e:
get_logger().error("bad literal", e)
def __ror__(self, other):
"""
RECURSIVE COALESCE OF DATA PROPERTIES
"""
if not _get(other, CLASS) in data_types:
get_logger().error("Expecting Data")
return to_data(other).__or__(self)
def literal_field(field):
"""
RETURN SAME WITH DOTS (`.`) ESCAPED
"""
try:
return field.replace(".", "\\.")
except Exception as e:
get_logger().error("bad literal", e)
def _all_default(d, default, seen=None):
"""
ANY VALUE NOT SET WILL BE SET BY THE default
THIS IS RECURSIVE
"""
if default is None:
return
if _get(default, CLASS) is Data:
default = object.__getattribute__(default,
SLOT) # REACH IN AND GET THE dict
# Log = _late_import()
# Log.error("strictly dict (or object) allowed: got {{type}}", type=_get(default, CLASS).__name__)
for k, default_value in default.items():
default_value = unwrap(
default_value
) # TWO DIFFERENT Dicts CAN SHARE id() BECAUSE THEY ARE SHORT LIVED
if is_data(d):
existing_value = d.get(k)
else:
existing_value = _get_attr(d, [k])
if existing_value == None:
if default_value != None:
if _get(default_value, CLASS) in data_types:
df = seen.get(id(default_value))
if df is not None:
_set_attr(d, [k], df)
else:
copy_dict = {}
seen[id(default_value)] = copy_dict
_set_attr(d, [k], copy_dict)
_all_default(copy_dict, default_value, seen)
else:
# ASSUME PRIMITIVE (OR LIST, WHICH WE DO NOT COPY)
try:
_set_attr(d, [k], default_value)
except Exception as e:
if PATH_NOT_FOUND not in e:
get_logger().error(
"Can not set attribute {{name}}",
name=k,
cause=e)
elif is_list(existing_value) or is_list(default_value):
_set_attr(d, [k], None)
_set_attr(d, [k],
listwrap(existing_value) + listwrap(default_value))
elif (hasattr(existing_value, "__setattr__")
or _get(existing_value, CLASS) in data_types) and _get(
default_value, CLASS) in data_types:
df = seen.get(id(default_value))
if df is not None:
_set_attr(d, [k], df)
else:
seen[id(default_value)] = existing_value
_all_default(existing_value, default_value, seen)
def __or__(self, other):
"""
RECURSIVE COALESCE OF DATA PROPERTIES
"""
if not _get(other, CLASS) in data_types:
get_logger().error("Expecting Data")
d = self._internal_dict
output = Data(**d) # COPY
output.__ior__(other)
return output
def unliteral_field(field):
"""
DUE TO PATHOLOGY IN MY CODE WE HAVE A path WITH ESCAPED DOTS BUT WE WANT OT USE IT ON A dict, NOT A Data
a = dict()
b = Data(a)
a[unliteral_field(k)]==b[k] (for all k)
:param field: THE STRING TO DE-literal IZE
:return: SIMPLER STRING
"""
if len(split_field(field)) > 1:
get_logger().error("Bad call! Dude!")
return field.replace("\\.", ".")
def unliteral_field(field):
"""
DUE TO PATHOLOGY IN MY CODE WE HAVE A path WITH ESCAPED DOTS BUT WE WANT OT USE IT ON A dict, NOT A Data
a = dict()
b = Data(a)
a[unliteral_field(k)]==b[k] (for all k)
:param field: THE STRING TO DE-literal IZE
:return: SIMPLER STRING
"""
if len(split_field(field)) > 1:
get_logger().error("Bad call! Dude!")
return field.replace("\\.", ".")
def _leaves_to_data(value):
"""
RETURN UNWRAPPED STRUCTURES
"""
if value == None:
return None
class_ = _get(value, CLASS)
if class_ in (text, binary_type, int, float):
return value
if class_ in data_types:
if class_ is Data:
value = from_data(value)
output = {}
for key, value in value.items():
value = _leaves_to_data(value)
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if is_binary(key):
key = key.decode("utf8")
d = output
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
else:
seq = split_field(key)
for k in seq[:-1]:
e = d.get(k, None)
if e is None:
d[k] = {}
e = d[k]
d = e
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return output
if hasattr(value, "__iter__"):
output = []
for v in value:
v = leaves_to_data(v)
output.append(v)
return output
return value
def _wrap_leaves(value):
if value == None:
return None
class_ = _get(value, CLASS)
if class_ in (text_type, binary_type, int, float):
return value
if class_ in data_types:
if class_ is Data:
value = unwrap(value)
output = {}
for key, value in value.items():
value = _wrap_leaves(value)
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if is_binary(key):
key = key.decode("utf8")
d = output
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
else:
seq = split_field(key)
for k in seq[:-1]:
e = d.get(k, None)
if e is None:
d[k] = {}
e = d[k]
d = e
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return output
if hasattr(value, '__iter__'):
output = []
for v in value:
v = wrap_leaves(v)
output.append(v)
return output
return value
def _wrap_leaves(value):
if value == None:
return None
if isinstance(value, (text_type, binary_type, int, float)):
return value
if isinstance(value, Mapping):
if isinstance(value, Data):
value = unwrap(value)
output = {}
for key, value in value.items():
value = _wrap_leaves(value)
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if isinstance(key, binary_type):
key = key.decode("utf8")
d = output
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
else:
seq = split_field(key)
for k in seq[:-1]:
e = d.get(k, None)
if e is None:
d[k] = {}
e = d[k]
d = e
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return output
if hasattr(value, '__iter__'):
output = []
for v in value:
v = wrap_leaves(v)
output.append(v)
return output
return value
def _all_default(d, default, seen=None):
"""
ANY VALUE NOT SET WILL BE SET BY THE default
THIS IS RECURSIVE
"""
if default is None:
return
if isinstance(default, Data):
default = object.__getattribute__(default,
"_dict") # REACH IN AND GET THE dict
# Log = _late_import()
# Log.error("strictly dict (or object) allowed: got {{type}}", type=default.__class__.__name__)
for k, default_value in default.items():
default_value = unwrap(
default_value
) # TWO DIFFERENT Dicts CAN SHARE id() BECAUSE THEY ARE SHORT LIVED
existing_value = _get_attr(d, [k])
if existing_value == None:
if default_value != None:
if isinstance(default_value, Mapping):
df = seen.get(id(default_value))
if df is not None:
_set_attr(d, [k], df)
else:
copy_dict = {}
seen[id(default_value)] = copy_dict
_set_attr(d, [k], copy_dict)
_all_default(copy_dict, default_value, seen)
else:
# ASSUME PRIMITIVE (OR LIST, WHICH WE DO NOT COPY)
try:
_set_attr(d, [k], default_value)
except Exception, e:
if PATH_NOT_FOUND not in e:
get_logger().error(
"Can not set attribute {{name}}",
name=k,
cause=e)
elif isinstance(existing_value, list) or isinstance(
default_value, list):
_set_attr(d, [k],
listwrap(existing_value) + listwrap(default_value))
def _wrap_leaves(value):
if value == None:
return None
if isinstance(value, (basestring, int, float)):
return value
if isinstance(value, Mapping):
if isinstance(value, Data):
value = unwrap(value)
output = {}
for key, value in value.iteritems():
value = _wrap_leaves(value)
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if isinstance(key, str):
key = key.decode("utf8")
d = output
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
else:
seq = split_field(key)
for k in seq[:-1]:
e = d.get(k, None)
if e is None:
d[k] = {}
e = d[k]
d = e
if value == None:
d.pop(seq[-1], None)
else:
d[seq[-1]] = value
return output
if hasattr(value, '__iter__'):
output = []
for v in value:
v = wrap_leaves(v)
output.append(v)
return output
return value
def _all_default(d, default, seen=None):
"""
ANY VALUE NOT SET WILL BE SET BY THE default
THIS IS RECURSIVE
"""
if default is None:
return
if isinstance(default, Data):
default = object.__getattribute__(default, SLOT) # REACH IN AND GET THE dict
# Log = _late_import()
# Log.error("strictly dict (or object) allowed: got {{type}}", type=default.__class__.__name__)
for k, default_value in default.items():
default_value = unwrap(default_value) # TWO DIFFERENT Dicts CAN SHARE id() BECAUSE THEY ARE SHORT LIVED
existing_value = _get_attr(d, [k])
if existing_value == None:
if default_value != None:
if isinstance(default_value, Mapping):
df = seen.get(id(default_value))
if df is not None:
_set_attr(d, [k], df)
else:
copy_dict = {}
seen[id(default_value)] = copy_dict
_set_attr(d, [k], copy_dict)
_all_default(copy_dict, default_value, seen)
else:
# ASSUME PRIMITIVE (OR LIST, WHICH WE DO NOT COPY)
try:
_set_attr(d, [k], default_value)
except Exception as e:
if PATH_NOT_FOUND not in e:
get_logger().error("Can not set attribute {{name}}", name=k, cause=e)
elif isinstance(existing_value, list) or isinstance(default_value, list):
_set_attr(d, [k], None)
_set_attr(d, [k], listwrap(existing_value) + listwrap(default_value))
elif (hasattr(existing_value, "__setattr__") or isinstance(existing_value, Mapping)) and isinstance(default_value, Mapping):
df = seen.get(id(default_value))
if df is not None:
_set_attr(d, [k], df)
else:
seen[id(default_value)] = existing_value
_all_default(existing_value, default_value, seen)
def _set_attr(obj, path, value):
obj = _get_attr(obj, path[:-1])
if obj is None: # DELIBERATE, WE DO NOT WHAT TO CATCH Null HERE (THEY CAN BE SET)
get_logger().error(PATH_NOT_FOUND)
attr_name = path[-1]
# ACTUAL SETTING OF VALUE
try:
old_value = _get_attr(obj, [attr_name])
if old_value == None:
old_value = None
new_value = value
else:
new_value = old_value.__class__(
value) # TRY TO MAKE INSTANCE OF SAME CLASS
except Exception, e:
old_value = None
new_value = value
def __setitem__(self, key, value):
if key == "":
get_logger().error("key is empty string. Probably a bad idea")
if key == None:
return Null
if key == ".":
# SOMETHING TERRIBLE HAPPENS WHEN value IS NOT A Mapping;
# HOPEFULLY THE ONLY OTHER METHOD RUN ON self IS from_data()
v = from_data(value)
_set(self, SLOT, v)
return v
try:
d = self._internal_dict
value = from_data(value)
if key.find(".") == -1:
if value is None:
d.pop(key, None)
else:
d[key] = value
return self
seq = _split_field(key)
for k in seq[:-1]:
d = _getdefault(d, k)
if value == None:
try:
d.pop(seq[-1], None)
except Exception as _:
pass
elif d == None:
d[literal_field(seq[-1])] = value
elif is_sequence(d):
for dd in d:
from_data(dd)[seq[-1]] = value
else:
d[seq[-1]] = value
return self
except Exception as e:
from mo_logs import Log
Log.error("can not set key={{key}}", key=key, cause=e)
def get_attr(obj, path):
"""
SAME AS object.__getattr__(), BUT USES DOT-DELIMITED path
"""
try:
return _get_attr(obj, split_field(path))
except Exception as e:
Log = get_logger()
if PATH_NOT_FOUND in e:
Log.error(PATH_NOT_FOUND + ": {{path}}", path=path, cause=e)
else:
Log.error("Problem setting value", e)
def __ior__(self, other):
"""
RECURSIVE COALESCE OF DATA PROPERTIES
"""
if not _get(other, CLASS) in data_types:
get_logger().error("Expecting Data")
d = self._internal_dict
for ok, ov in other.items():
if ov == None:
continue
sv = d.get(ok)
if sv == None:
d[ok] = ov
elif isinstance(sv, Data):
sv |= ov
elif is_data(sv):
wv = _new(Data)
_set(wv, SLOT, sv)
wv |= ov
return self
def leaves(value, prefix=None):
"""
LIKE items() BUT RECURSIVE, AND ONLY FOR THE LEAVES (non dict) VALUES
SEE leaves_to_data FOR THE INVERSE
:param value: THE Mapping TO TRAVERSE
:param prefix: OPTIONAL PREFIX GIVEN TO EACH KEY
:return: Data, WHICH EACH KEY BEING A PATH INTO value TREE
"""
prefix = coalesce(prefix, "")
output = []
for k, v in value.items():
try:
if _get(v, CLASS) in data_types:
output.extend(leaves(v,
prefix=prefix + literal_field(k) + "."))
else:
output.append((prefix + literal_field(k), from_data(v)))
except Exception as e:
get_logger().error("Do not know how to handle", cause=e)
return output
def get_attr(obj, path):
"""
SAME AS object.__getattr__(), BUT USES DOT-DELIMITED path
"""
try:
return _get_attr(obj, split_field(path))
except Exception as e:
Log = get_logger()
if PATH_NOT_FOUND in e:
Log.error(PATH_NOT_FOUND+": {{path}}", path=path, cause=e)
else:
Log.error("Problem setting value", e)
def _iadd(self, other):
"""
RECURSIVE ADDITION OF DATA PROPERTIES
* LISTS ARE CONCATENATED
* SETS ARE UNIONED
* NUMBERS ARE ADDED
"""
if not _get(other, CLASS) in data_types:
get_logger().error("Expecting Data")
d = from_data(self)
for ok, ov in other.items():
sv = d.get(ok)
if sv == None:
d[ok] = deepcopy(ov)
elif isinstance(ov, (Decimal, float, long, int)):
if _get(sv, CLASS) in data_types:
get_logger().error(
"can not add {{stype}} with {{otype}",
stype=_get(sv, CLASS).__name__,
otype=_get(ov, CLASS).__name__,
)
elif is_list(sv):
d[ok].append(ov)
else:
d[ok] = sv + ov
elif is_list(ov):
d[ok] = listwrap(sv) + ov
elif _get(ov, CLASS) in data_types:
if _get(sv, CLASS) in data_types:
_iadd(sv, ov)
elif is_list(sv):
d[ok].append(ov)
else:
get_logger().error(
"can not add {{stype}} with {{otype}",
stype=_get(sv, CLASS).__name__,
otype=_get(ov, CLASS).__name__,
)
else:
if _get(sv, CLASS) in data_types:
get_logger().error(
"can not add {{stype}} with {{otype}",
stype=_get(sv, CLASS).__name__,
otype=_get(ov, CLASS).__name__,
)
else:
d[ok].append(ov)
return self
def set_attr(obj, path, value):
"""
SAME AS object.__setattr__(), BUT USES DOT-DELIMITED path
RETURN OLD VALUE
"""
try:
return _set_attr(obj, split_field(path), value)
except Exception as e:
Log = get_logger()
if PATH_NOT_FOUND in e:
Log.warning(PATH_NOT_FOUND + ": {{path}}", path=path, cause=e)
else:
Log.error("Problem setting value", cause=e)
def set_attr(obj, path, value):
"""
SAME AS object.__setattr__(), BUT USES DOT-DELIMITED path
RETURN OLD VALUE
"""
try:
return _set_attr(obj, split_field(path), value)
except Exception as e:
Log = get_logger()
if PATH_NOT_FOUND in e:
Log.warning(PATH_NOT_FOUND + ": {{path}}", path=path, cause=e)
else:
Log.error("Problem setting value", cause=e)
def _set_attr(obj_, path, value):
obj = _get_attr(obj_, path[:-1])
if (
obj is None
): # DELIBERATE USE OF `is`: WE DO NOT WHAT TO CATCH Null HERE (THEY CAN BE SET)
obj = _get_attr(obj_, path[:-1])
if obj is None:
get_logger().error(PATH_NOT_FOUND +
" tried to get attribute of None")
attr_name = path[-1]
# ACTUAL SETTING OF VALUE
try:
old_value = _get_attr(obj, [attr_name])
old_type = _get(old_value, CLASS)
if old_value == None or old_type in (bool, int, float, text,
binary_type):
old_value = None
new_value = value
elif value == None:
new_value = None
else:
new_value = _get(old_value, CLASS)(
value) # TRY TO MAKE INSTANCE OF SAME CLASS
except Exception:
old_value = None
new_value = value
try:
setattr(obj, attr_name, new_value)
return old_value
except Exception as e:
try:
obj[attr_name] = new_value
return old_value
except Exception as f:
get_logger().error(PATH_NOT_FOUND, cause=[f, e])
def clear(self):
get_logger().error("clear() not supported")
else:
new_value = old_value.__class__(
value) # TRY TO MAKE INSTANCE OF SAME CLASS
except Exception, e:
old_value = None
new_value = value
try:
setattr(obj, attr_name, new_value)
return old_value
except Exception, e:
try:
obj[attr_name] = new_value
return old_value
except Exception, f:
get_logger().error(PATH_NOT_FOUND)
def lower_match(value, candidates):
return [v for v in candidates if v.lower() == value.lower()]
def wrap(v):
type_ = _get(v, "__class__")
if type_ is dict:
m = Data(v)
return m
# m = object.__new__(Data)
# object.__setattr__(m, "_dict", v)
# return m