class TimedObjCache(Cache):
def __init__(self, expiration=3600, file=None):
super().__init__(file)
self._data = OrderedDict()
self.expiration = expiration
def _prune(self):
old = []
for key in self._data.keys():
data, added = self._data[key]
time_since = time() - added
if time_since >= self.expiration:
old.append((key, data))
del self._data[key]
else:
break
return old
def get(self, key):
self._prune()
if key in self._data.keys():
return self._data[key][0]
return None
def store(self, key, data):
self._data[key] = (data, time())
def data(self):
return self._data
def __iter__(self):
self._data.__iter__()
class TimedObjCache(Cache): def __init__(self, expiration=3600, file=None): super().__init__(file) self._data = OrderedDict() self.expiration = expiration def _prune(self): old = [] for key in self._data.keys(): data, added = self._data[key] time_since = time() - added if time_since >= self.expiration: old.append((key, data)) del self._data[key] else: break return old def get(self, key): self._prune() if key in self._data.keys(): return self._data[key][0] return None def store(self, key, data): self._data[key] = (data, time()) def data(self): return self._data def __iter__(self): self._data.__iter__()
class AccountManagement(object):
def __init__(self, ac_path='', cookies_dir=''):
self.accounts = OrderedDict()
self.load(ac_path, cookies_dir)
self._iter_ = self.accounts.__iter__()
def set_cookies_dir(self, dir):
for ac in self.accounts.values():
ac.set_cookies_dir(dir)
def load(self, ac_path='', cookies_dir=''):
if os.path.isfile(ac_path):
with open(ac_path) as f:
for ip in f.read().split():
id, pwd = ip.split('----')
account = Account(id, pwd, cookies_dir)
account.loadCookiesList()
self.accounts[id] = account
def save(self):
for ac in self.accounts.values():
ac.save_cookies()
def __iter__(self):
return self.accounts.__iter__()
def next(self):
v = None
try:
v = next(self._iter_)
except Exception as e:
self._iter_ = self.accounts.__iter__()
print(e)
return self.accounts.get(v)
class FifoCache():
def __init__(self, max_capacity):
self.max_capacity = max_capacity
self.store = OrderedDict()
def iterkeys(self):
return self.store.iterkeys()
def itervalues(self):
return self.store.itervalues()
def iteritems(self):
return self.store.iteritems()
def __getitem__(self, key):
return self.store[key]
def __setitem__(self, key, value):
if key in self.store:
del self.store[key]
else:
if len(self.store) == self.max_capacity:
self.store.popitem(last=False) # last=False --> FIFO, last=True --> LIFO
self.store[key] = value
def __delitem__(self, key):
del self.store[key]
def __iter__(self):
return self.store.__iter__()
def __len__(self):
return len(self.store)
class Cache(MutableMapping):
def __init__(self, maxlen, items=None):
self._maxlen = maxlen
self.d = OrderedDict()
if items:
for k, v in items:
self[k] = v
@property
def maxlen(self):
return self._maxlen
def __getitem__(self, key):
self.d.move_to_end(key)
return self.d[key]
def __setitem__(self, key, value):
if key in self.d:
self.d.move_to_end(key)
elif len(self.d) == self.maxlen:
self.d.popitem(last=False)
self.d[key] = value
def __delitem__(self, key):
del self.d[key]
def __iter__(self):
return self.d.__iter__()
def __len__(self):
return len(self.d)
class IndexDictionary(object):
"""
A very basic indexing mechanism object that assigns consecutive indexes to the indexed objects.
"""
def __init__(self, elements=None):
self.data = OrderedDict()
elements = [] if elements is None else elements
for element in elements:
self.add(element)
def get_index(self, key):
return self.data[key]
def add(self, obj):
if obj in self.data:
raise RuntimeError("Duplicate element '{}'".format(obj))
self.data[obj] = len(self.data)
def dump(self):
return [str(o) for o in self.data.keys()]
def __str__(self):
return ','.join('{}: {}'.format(k, o) for k, o in self.data.items())
__repr__ = __str__
def __iter__(self):
return self.data.__iter__()
def __contains__(self, k):
return k in self.data
def __len__(self):
return len(self.data)
class Namelist(object):
def __init__(self, nl_obj: dict = None):
self._name = type(self).__name__
if nl_obj is None:
self._nl = OrderedDict()
else:
self._nl = nl_obj
def __iter__(self):
return self._nl.__iter__()
def __len__(self):
acc = 0
for v in self._nl.values():
acc += len(v)
return acc
def keys(self):
return self._nl.keys()
def __str__(self):
return nml2str(self._nl)
def write(self, file_path, grp_names: list = []):
with open(file_path, "w") as f:
f.write(nml2str(self._nl, grp_names=grp_names))
class IndexDictionary(object):
"""
A very basic indexing mechanism object that assigns consecutive indexes to the indexed objects.
"""
def __init__(self, elements=None):
self.data = OrderedDict()
elements = [] if elements is None else elements
for element in elements:
self.add(element)
def get_index(self, key):
return self.data[key]
def add(self, obj):
if obj in self.data:
raise RuntimeError("Duplicate element '{}'".format(obj))
self.data[obj] = len(self.data)
def dump(self):
return [str(o) for o in self.data.keys()]
def __str__(self):
return ','.join('{}: {}'.format(k, o) for k, o in self.data.items())
__repr__ = __str__
def __iter__(self):
return self.data.__iter__()
def __contains__(self, k):
return k in self.data
def __len__(self):
return len(self.data)
class FifoCache():
def __init__(self, max_capacity):
self.max_capacity = max_capacity
self.store = OrderedDict()
def iterkeys(self):
return self.store.iterkeys()
def itervalues(self):
return self.store.itervalues()
def iteritems(self):
return self.store.iteritems()
def __getitem__(self, key):
return self.store[key]
def __setitem__(self, key, value):
if key in self.store:
del self.store[key]
else:
if len(self.store) == self.max_capacity:
self.store.popitem(
last=False) # last=False --> FIFO, last=True --> LIFO
self.store[key] = value
def __delitem__(self, key):
del self.store[key]
def __iter__(self):
return self.store.__iter__()
def __len__(self):
return len(self.store)
class Level(object):
def __init__(self):
self.level=OrderedDict()
def __iter__(self):
return self.level.__iter__()
def __getitem__(self,order_id):
return self.level[order_id]
def __len__(self):
return len(self.level)
def __contains__(self,key):
return key in self.level
def delete(self,order_id):
del self.level[order_id]
def insert(self,order):
self.level[order.order_id]=order
# note that this will return orders sorted by insertion time, not
# orders.creation_time
def values(self):
return self.level.values()
class LabBook:
_singlenton = None
def __new__(cls, *args, **kwargs):
if not cls._singlenton:
cls._singlenton = super(LabBook, cls).__new__(cls, *args, **kwargs)
return cls._singlenton
def __init__(self):
trackers_book = TrackerBook()
self.labsByKey = OrderedDict((trackers_book[item].keystone, trackers_book[item])
for item in trackers_book if type(trackers_book[item]) == Lab)
self.labsByName = OrderedDict((trackers_book[item].name, trackers_book[item])
for item in trackers_book if type(trackers_book[item]) == Lab)
def __getitem__(self, item):
if item in self.labsByKey:
return self.labsByKey[item]
if item in self.labsByName:
return self.labsByName[item]
raise KeyError
def __iter__(self):
return self.labsByName.__iter__()
@property
def labs(self):
return list(self.labsByName.keys())
def __len__(self):
return len(self.labsByKey)
class TargetVars(Mapping):
"""Immutable ordered mapping from target variables to their values."""
EMPTY = EmptyTargetVar()
def __init__(self, target_vars=tuple(), is_empty=True):
if is_empty:
target_vars = [(v, self.EMPTY) for v in target_vars]
self._od = OrderedDict(target_vars)
# getitem, len, iter wrap OrderedDict behavior
def __getitem__(self, k): return self._od.__getitem__(k)
def __len__(self): return self._od.__len__()
def __iter__(self): return self._od.__iter__()
def update(self, *args, **kwargs):
cpy = self.copy()
cpy._od.update(*args, **kwargs)
return cpy
def copy(self):
return self.__class__(self._od)
def __str__(self):
"""Format target vars for printing"""
if len(self) > 1: return "({})".format(", ".join(self._od.keys()))
else: return "".join(self._od.keys())
def defined_items(self):
"""Return copy of instance, omitting entries that are EMPTY"""
return self.__class__([(k, v) for k,v in self.items() if v is not self.EMPTY], is_empty=False)
class LRU:
def __init__(self, capacity):
self.dic = OrderedDict()
self.vacancy = capacity
self.next_val = 256
def get(self, key):
if key not in self.dic:
return None
self.dic.move_to_end(key, last=False) # move to the front
return self.dic[key]
def set(self, key):
if key in self.dic:
self.dic.move_to_end(key, last=False) # move to the front
else:
value = self.next_val
if self.vacancy > 0:
self.vacancy -= 1
self.next_val += 1
else:
(key,
val) = self.dic.popitem(last=True) # pop the last element
# reuse popped value
value = val
self.dic[key] = value # set new val
self.dic.move_to_end(key, last=False) # move to the front
def __iter__(self):
return self.dic.__iter__()
class DirectionStatisticsContainer:
def __init__(self, initial_directions):
self.directions_dict = OrderedDict()
self.directions_arr = []
for index, dir in enumerate(initial_directions):
self.directions_dict[tuple(dir)] = _SingleDirectionStatistics(dir)
self.directions_arr.append(self.directions_dict[tuple(dir)])
def __getitem__(self, key):
return self.directions_dict[tuple(key)]
def __setitem__(self, key, value):
self.directions_dict[tuple(key)] = value
def __iter__(self):
return self.directions_dict.__iter__()
def __str__(self):
return str(self.directions_dict)
def to_dict(self):
return [{
"dir": dir,
"stats": self.directions_dict[dir].to_dict()
} for dir in self.directions_dict]
class LRUCache(MutableMapping):
"""A least-recently-used cache.
Implementation stolen from `this article`_.
.. _this article: https://www.kunxi.org/blog/2014/05/lru-cache-in-python/
"""
def __init__(self, capacity):
self.capacity = capacity
self.cache = OrderedDict()
def __getitem__(self, key):
value = self.cache.pop(key)
self.cache[key] = value # move value to the head of the dict
return value
def __setitem__(self, key, value):
try:
self.cache.pop(key)
except KeyError:
if len(self.cache) >= self.capacity:
self.cache.popitem(last=False)
self.cache[key] = value
def __delitem__(self, key):
del self.cache[key]
def __iter__(self):
return self.cache.__iter__()
def __len__(self):
return self.cache.__len__()
class Metadata(MutableMapping):
def __init__(self, seq=None):
self.dct = OrderedDict(seq) if seq else OrderedDict()
def __contains__(self, key):
return self.dct.__contains__(key)
def __getitem__(self, key):
return self.dct.__getitem__(key)
def __setitem__(self, key, value):
self.dct.__setitem__(key, value)
def __delitem__(self, key):
return self.dct.__delitem__(key)
def __iter__(self):
return self.dct.__iter__()
def __len__(self):
return self.dct.__len__()
def __repr__(self):
return repr(self.dct)
def __str__(self):
return str(self.dct)
class Metadata(MutableMapping):
def __init__(self, seq=None):
self.dct = OrderedDict(seq) if seq else OrderedDict()
def __contains__(self, key):
return self.dct.__contains__(key)
def __getitem__(self, key):
return self.dct.__getitem__(key)
def __setitem__(self, key, value):
self.dct.__setitem__(key, value)
def __delitem__(self, key):
return self.dct.__delitem__(key)
def __iter__(self):
return self.dct.__iter__()
def __len__(self):
return self.dct.__len__()
def __repr__(self):
return repr(self.dct)
def __str__(self):
return str(self.dct)
class OrderedSet:
def __init__(self, values):
self.s = OrderedDict()
for v in values:
self.s[v] = None
def contains(self, value):
return value in self.s
def add(self, value):
self.s[value] = None
def union(self, rhs):
result = OrderedSet([])
for v in self.s:
result.add(v)
for v in rhs.s:
result.add(v)
return result
def __or__(self, rhs):
return self.union(rhs)
def __iter__(self):
return self.s.__iter__()
def __next__(self):
return self.s.next()
def __eq__(self, rhs):
return self.s == rhs.s
def __len__(self):
return len(self.s)
class LRUCache:
def __init__(self, capacity: int):
self.capacity = capacity
self.cache = OrderedDict()
def get(self, key: int) -> int:
val = self.cache.get(key, None)
if val:
pop_val = self.cache.pop(key)
self.cache[key] = pop_val
return val if val else -1
def put(self, key: int, value: int) -> None:
if self.cache.__len__() + 1 <= self.capacity:
if key in self.cache:
self.cache.pop(key)
self.cache[key] = value
else:
self.cache[key] = value
else:
if key in self.cache:
self.cache.pop(key)
self.cache[key] = value
else:
self.cache.pop(self.cache.__iter__().__next__())
self.cache[key] = value
def relabel(self, keys):
""" Relabel a collection of keys (a dictionary of old -> new.
This does not allow moving things between different directories.
This code seems useless. The purpose of relabeling is to move around
files across directories. """
assert(all(self.key_exists(k) and not self.key_exists(v) \
for k,v in keys.iteritems()))
for old_key in keys.keys():
new_key = keys[old_key]
old_parent, old_file = self.__get_parent_dir_and_file(old_key)
new_parent, new_file = self.__get_parent_dir_and_file(new_key)
if old_parent != new_parent:
raise KeyError("key parent directory mismatch:", \
old_key," and ", new_key)
old_index, old_key_file = self.__find_enclosing_index(old_key, False)
l = old_index._OrderedDict__map[old_file]
del old_index._OrderedDict__map[old_file]
old_index._OrderedDict__map[new_file] = l
l[2] = new_file
dict.__setitem__(old_index, new_file, dict.pop(old_index, old_file))
self._list = [k for k in OrderedDict.__iter__(self)]
self._full_key_list = None
def relabel(self, keys):
""" Relabel a collection of keys (a dictionary of old -> new.
This does not allow moving things between different directories.
This code seems useless. The purpose of relabeling is to move around
files across directories. """
assert(all(self.key_exists(k) and not self.key_exists(v) \
for k,v in keys.iteritems()))
for old_key in keys.keys():
new_key = keys[old_key]
old_parent, old_file = self.__get_parent_dir_and_file(old_key)
new_parent, new_file = self.__get_parent_dir_and_file(new_key)
if old_parent != new_parent:
raise KeyError("key parent directory mismatch:", \
old_key," and ", new_key)
old_index, old_key_file = self.__find_enclosing_index(
old_key, False)
l = old_index._OrderedDict__map[old_file]
del old_index._OrderedDict__map[old_file]
old_index._OrderedDict__map[new_file] = l
l[2] = new_file
dict.__setitem__(old_index, new_file,
dict.pop(old_index, old_file))
self._list = [k for k in OrderedDict.__iter__(self)]
self._full_key_list = None
class OrderedSet(MutableSet):
def __init__(self, sequence=None):
super().__init__()
if sequence is None:
self._data = OrderedDict()
else:
kwargs = {v: 1 for v in sequence}
self._data = OrderedDict(**kwargs)
def __contains__(self, item):
"""Override."""
return self._data.__contains__(item)
def __iter__(self):
"""Override."""
return self._data.__iter__()
def __len__(self):
"""Override."""
return self._data.__len__()
def add(self, item):
"""Override."""
self._data.__setitem__(item, 1)
def discard(self, item):
"""Override."""
if item in self._data:
self._data.__delitem__(item)
def __repr__(self):
return f"{self.__class__.__name__}({list(self._data.keys())})"
class TargetVars(Mapping):
"""Immutable ordered mapping from target variables to their values."""
EMPTY = EmptyTargetVar()
def __init__(self, target_vars=tuple(), is_empty=True):
if is_empty:
target_vars = [(v, self.EMPTY) for v in target_vars]
self._od = OrderedDict(target_vars)
# getitem, len, iter wrap OrderedDict behavior
def __getitem__(self, k): return self._od.__getitem__(k)
def __len__(self): return self._od.__len__()
def __iter__(self): return self._od.__iter__()
def update(self, *args, **kwargs):
cpy = self.copy()
cpy._od.update(*args, **kwargs)
return cpy
def copy(self):
return self.__class__(self._od)
def __str__(self):
"""Format target vars for printing"""
if len(self) > 1: return "({})".format(", ".join(self._od.keys()))
else: return "".join(self._od.keys())
def defined_items(self):
"""Return copy of instance, omitting entries that are EMPTY"""
return self.__class__([(k, v) for k,v in self.items() if v is not self.EMPTY], is_empty=False)
class View(object):
""" Presents a view of the set of `Option` arguments to a search command
TODO: Description
"""
def __init__(self, command):
self._items = OrderedDict([
(option.name, Option.Item(command, option))
for member_name, option in type(command).option_definitions
])
return
def __contains__(self, name):
return name in self._items
def __getitem__(self, name):
return self._items[name]
def __iter__(self):
return self._items.__iter__()
def __len__(self):
return len(self._items)
def __repr__(self):
text = ''.join([
'Option.View(',
','.join([repr(item) for item in self.itervalues()]), ')'
])
return text
def __str__(self):
text = ' '.join(
[str(item) for item in self.itervalues() if item.is_set])
return text
#region Methods
def get_missing(self):
missing = [
item.name for item in self._items.itervalues()
if item.is_required and not item.is_set
]
return missing if len(missing) > 0 else None
def iteritems(self):
return self._items.iteritems()
def iterkeys(self):
return self.__iter__()
def itervalues(self):
return self._items.itervalues()
def reset(self):
for value in self.itervalues():
value.reset()
return
class DictContainer(MutableMapping):
"""
Wrapper around a dict, to track the elements' parents.
**This class shouldn't be instantiated directly by users,
but by the elements that contain it**.
:param args: elements contained in the dict--like object
:param oktypes: type or tuple of types that are allowed as items
:type oktypes: ``type`` | ``tuple``
:param parent: the parent element
:type parent: ``Element``
"""
__slots__ = ['dict', 'oktypes', 'parent', 'location']
def __init__(self, *args, oktypes=object, parent=None, **kwargs):
self.oktypes = oktypes
self.parent = parent
self.location = None
self.dict = OrderedDict()
self.update(args) # Must be a sequence of tuples
self.update(kwargs) # Order of kwargs is not preserved
def __contains__(self, item):
return item in self.dict
def __len__(self):
return len(self.dict)
def __getitem__(self, k):
return attach(self.dict[k], self.parent, self.location)
def __delitem__(self, k):
del self.dict[k]
def __setitem__(self, k, v):
v = check_type(v, self.oktypes)
self.dict[k] = v
def __str__(self):
return self.__repr__()
def __repr__(self):
return 'DictContainer({})'.format(' '.join(repr(x) for x in self.dict))
def __iter__(self):
return self.dict.__iter__()
def to_json(self):
items = self.dict.items()
return OrderedDict((k, to_json_wrapper(v)) for k, v in items)
# return [item.to_json() for item in self.dict]
def walk(self, action, doc):
ans = [(k, v.walk(action, doc)) for k, v in self.items()]
ans = [(k, v) for k, v in ans if v != []]
return ans
class OrderedDictEx(MutableMapping):
def __init__(self, fifo=True, maxSize=None, readImpactsOrder=False):
super(OrderedDictEx, self).__init__()
self._dic = OrderedDict()
self.fifo = fifo
self.max_size = maxSize
self.read_impacts_order = readImpactsOrder
self._recursing = False
self._lock = RLock()
self.recurse_count = 0
def __setitem__(self, key, value):
with self._lock:
# Store items in order that key was last added.
# This will ensure update has same impact on position
# as setting the item.
skipSizeCheck = False # optimization
if key in self._dic:
del self._dic[key]
skipSizeCheck = True
self._dic[key] = value
if self.max_size is not None and not skipSizeCheck:
while len(self._dic) > self.max_size:
self.removeOrderedItem()
def __str__(self):
return str(self._dic)
def __unicode__(self):
return unicode(self._dic)
def __repr__(self):
return repr(self._dic)
def __len__(self):
return len(self._dic)
def __getitem__(self, key):
with self._lock:
val = self._dic[key]
if self.read_impacts_order:
self.__setitem__(key, val)
return val
def __delitem__(self, key):
del self._dic[key]
def __iter__(self):
return self._dic.__iter__()
def removeOrderedItem(self):
return self._dic.popitem(not self.fifo)
class OrderedDictEx(MutableMapping):
def __init__(self, fifo=True, maxSize=None, readImpactsOrder=False):
super(OrderedDictEx,self).__init__()
self._dic = OrderedDict()
self.fifo = fifo
self.max_size = maxSize
self.read_impacts_order = readImpactsOrder
self._recursing = False
self._lock = RLock()
self.recurse_count = 0
def __setitem__(self, key, value):
with self._lock:
# Store items in order that key was last added.
# This will ensure update has same impact on position
# as setting the item.
skipSizeCheck = False # optimization
if key in self._dic:
del self._dic[key]
skipSizeCheck = True
self._dic[key] = value
if self.max_size is not None and not skipSizeCheck:
while len(self._dic) > self.max_size:
self.removeOrderedItem()
def __str__(self):
return str(self._dic)
def __unicode__(self):
return unicode(self._dic)
def __repr__(self):
return repr(self._dic)
def __len__(self):
return len(self._dic)
def __getitem__(self, key):
with self._lock:
val = self._dic[key]
if self.read_impacts_order:
self.__setitem__(key, val)
return val
def __delitem__(self, key):
del self._dic[key]
def __iter__(self):
return self._dic.__iter__()
def removeOrderedItem(self):
return self._dic.popitem(not self.fifo)
class View(object):
""" Presents a view of the set of `Option` arguments to a search command
TODO: Description
"""
def __init__(self, command):
self._items = OrderedDict([
(option.name, Option.Item(command, option))
for member_name, option in type(command).option_definitions])
return
def __contains__(self, name):
return name in self._items
def __getitem__(self, name):
return self._items[name]
def __iter__(self):
return self._items.__iter__()
def __len__(self):
return len(self._items)
def __repr__(self):
text = ''.join([
'Option.View(',
','.join([repr(item) for item in self.itervalues()]),
')'])
return text
def __str__(self):
text = ' '.join(
[str(item) for item in self.itervalues() if item.is_set])
return text
#region Methods
def get_missing(self):
missing = [
item.name for item in self._items.itervalues()
if item.is_required and not item.is_set]
return missing if len(missing) > 0 else None
def iteritems(self):
return self._items.iteritems()
def iterkeys(self):
return self.__iter__()
def itervalues(self):
return self._items.itervalues()
def reset(self):
for value in self.itervalues():
value.reset()
return
class CompoundValue(object):
def __init__(self, *args, **kwargs):
self.__values__ = OrderedDict()
# Set default values
for container_name, container in self._xsd_type.elements_nested:
values = container.default_value
if isinstance(container, Indicator):
self.__values__.update(values)
else:
self.__values__[container_name] = values
# Set attributes
for attribute_name, attribute in self._xsd_type.attributes:
self.__values__[attribute_name] = attribute.default_value
# Set elements
items = _process_signature(self._xsd_type, args, kwargs)
for key, value in items.items():
self.__values__[key] = value
def __contains__(self, key):
return self.__values__.__contains__(key)
def __len__(self):
return self.__values__.__len__()
def __iter__(self):
return self.__values__.__iter__()
def __repr__(self):
return PrettyPrinter().pformat(self.__values__)
def __delitem__(self, key):
return self.__values__.__delitem__(key)
def __getitem__(self, key):
return self.__values__[key]
def __setitem__(self, key, value):
self.__values__[key] = value
def __setattr__(self, key, value):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__setattr__(key, value)
self.__values__[key] = value
def __getattribute__(self, key):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__getattribute__(key)
try:
return self.__values__[key]
except KeyError:
raise AttributeError(
"%s instance has no attribute '%s'" % (
self.__class__.__name__, key))
class CompoundValue(object):
def __init__(self, *args, **kwargs):
self.__values__ = OrderedDict()
# Set default values
for container_name, container in self._xsd_type.elements_nested:
values = container.default_value
if isinstance(container, Indicator):
self.__values__.update(values)
else:
self.__values__[container_name] = values
# Set attributes
for attribute_name, attribute in self._xsd_type.attributes:
self.__values__[attribute_name] = attribute.default_value
# Set elements
items = _process_signature(self._xsd_type, args, kwargs)
for key, value in items.items():
self.__values__[key] = value
def __contains__(self, key):
return self.__values__.__contains__(key)
def __len__(self):
return self.__values__.__len__()
def __iter__(self):
return self.__values__.__iter__()
def __repr__(self):
return PrettyPrinter().pformat(self.__values__)
def __delitem__(self, key):
return self.__values__.__delitem__(key)
def __getitem__(self, key):
return self.__values__[key]
def __setitem__(self, key, value):
self.__values__[key] = value
def __setattr__(self, key, value):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__setattr__(key, value)
self.__values__[key] = value
def __getattribute__(self, key):
if key.startswith('__') or key in ('_xsd_type', '_xsd_elm'):
return super(CompoundValue, self).__getattribute__(key)
try:
return self.__values__[key]
except KeyError:
raise AttributeError(
"%s instance has no attribute '%s'" % (
self.__class__.__name__, key))
class ShiftReg595():
def __init__(self, ser, oe, rclk, srclk, names=["a", "b", "c", "d", "e", "f", "g", "h"]):
self.pins = Collector(ser=ser, oe=oe, rclk=rclk, srclk=srclk)
self.names = OrderedDict((i, 0) for i in names[::-1])
# initialize GPIO
GPIO.setwarnings(False)
GPIO.setmode(GPIO.BCM)
for i in self.pins:
GPIO.setup(self.pins[i], GPIO.OUT)
self.set_all(0)
self.output_enable(1)
def __getitem__(self, key):
return self.name[key]
def __iter__(self):
return self.names.__iter__()
def __setitem__(self, key, value):
self.names[key] = value
self.update()
def driver(self):
GPIO.output(self.pins.rclk, 1)
sleep(0.000000094)
GPIO.output(self.pins.rclk, 0)
def output_enable(self, en):
GPIO.output(self.pins.oe, not en)
# sets value to be shifted but does not shift the value into the register
def place(self, key, value):
self.names[key] = value
def set_all(self, inp):
for i in self.names:
self.names[i] = inp
self.shift_in(self.names[i])
self.driver()
def shift_in(self, inp):
# shift a zero in
GPIO.output(self.pins.ser, inp)
sleep(0.000000125)
GPIO.output(self.pins.srclk, 1)
sleep(0.0000001)
GPIO.output(self.pins.srclk, 0)
def update(self):
for i in self.names:
self.shift_in(self.names[i])
self.driver()
def __getLine(self, pos=None, add_time=False):
if add_time is False:
line = ()
else:
line = (self._time.getTimeFromPos(pos), )
for var in OrderedDict.__iter__(self):
line += (OrderedDict.__getitem__(self, var)[pos],)
return line
class ArgGroup(object):
'''
object representing group of args to be passed to ArgumentParser.add_argument
'''
# property that generates the *args that get passed to .add_argument
@property
def argsForAdd(self):
return [argProp.argsForAdd for argProp in self.values()]
# property that generates the **kwargs that get passed to .add_argument
@property
def kwargsForAdd(self):
return [argProp.kwargsForAdd for argProp in self.values()]
@property
def argNames(self):
return [argProp.name for argProp in self.values()]
# short aliases
@property
def argProps(self):
return self.argProperties
def __init__(self, name, *argProperties):
# init attrs from args
self.name = name
self.argProperties = OrderedDict()
self.addArgProperty(*argProperties)
def __contains__(self, key):
return key in self.argNames
def __iter__(self):
return self.argProperties.__iter__()
# other iterators
def keys(self):
return self.argProperties.keys()
def items(self):
return self.argProperties.items()
def values(self):
return self.argProperties.values()
def addArgProperty(self, *argProperties):
for argProp in argProperties:
self.argProperties[argProp.name] = argProp
def pop(self, name):
return self.argProperties.pop(name)
def update(self, other):
self.argProperties.update(other.argProperties)
class FileConfig(MutableMapping):
"""
Maps key-value pairs to a backing config file.
You can manually edit the file by modifying self.content.
"""
def __init__(self, filename, params=[]):
self.filename = filename
self.params = OrderedDict()
for param in params:
self.add_param(*param)
def read(self):
try:
with open(self.filename, 'r') as file:
self.content = unicode(file.read())
except IOError as e:
log.error(e)
self.content = u''
#Initialize all params from default values.
for key in self.params:
self[key] = self[key]
def commit(self):
if not os.path.isfile(self.filename):
dir = os.path.dirname(self.filename)
try:
os.makedirs(dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
with open(self.filename, 'w+') as file:
#Fix all linebreaks
file.write(os.linesep.join(self.content.splitlines()))
def add_param(self, key, handler):
self.params[key] = handler
def __iter__(self):
return self.params.__iter__()
def __len__(self):
return len(self.params)
def __getitem__(self, key):
return self.params[key].read(self.content)
def __setitem__(self, key, value):
self.content = self.params[key].write(self.content, value)
def keys(self):
return self.params.keys()
def __delitem__(self, key):
del self.params[key]
class MetafeatureFunctions(object):
def __init__(self):
self.functions = OrderedDict()
self.dependencies = OrderedDict()
self.values = OrderedDict()
def clear(self):
self.values = OrderedDict()
def __iter__(self):
return self.functions.__iter__()
def __getitem__(self, item):
return self.functions.__getitem__(item)
def __setitem__(self, key, value):
return self.functions.__setitem__(key, value)
def __delitem__(self, key):
return self.functions.__delitem__(key)
def __contains__(self, item):
return self.functions.__contains__(item)
def get_value(self, key):
return self.values[key].value
def set_value(self, key, item):
self.values[key] = item
def is_calculated(self, key):
"""Return if a helper function has already been executed.
Necessary as get_value() can return None if the helper function hasn't
been executed or if it returned None."""
return key in self.values
def get_dependency(self, name):
"""Return the dependency of metafeature "name".
"""
return self.dependencies.get(name)
def define(self, name, dependency=None):
"""Decorator for adding metafeature functions to a "dictionary" of
metafeatures. This behaves like a function decorating a function,
not a class decorating a function"""
def wrapper(metafeature_class):
instance = metafeature_class()
self.__setitem__(name, instance)
self.dependencies[name] = dependency
return instance
return wrapper
def addData(self, data):
"""
Adds data to the set. Must match the variable order of the set and the
number of variables in the set.
"""
if len(data) != 0:
pos = 1
for var in OrderedDict.__iter__(self):
(OrderedDict.__getitem__(self, var).
addData([(column[0], column[pos]) for column in data]))
pos += 1
class FileConfig(MutableMapping):
"""
Maps key-value pairs to a backing config file.
You can manually edit the file by modifying self.content.
"""
def __init__(self, filename, params=[]):
self.filename = filename
self.params = OrderedDict()
for param in params:
self.add_param(*param)
def read(self):
try:
with open(self.filename, 'r') as file:
self.content = unicode(file.read())
except IOError as e:
log.error(e)
self.content = u''
#Initialize all params from default values.
for key in self.params:
self[key] = self[key]
def commit(self):
if not os.path.isfile(self.filename):
dir = os.path.dirname(self.filename)
try:
os.makedirs(dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise e
with open(self.filename, 'w+') as file:
#Fix all linebreaks
file.write(os.linesep.join(self.content.splitlines()))
def add_param(self, key, handler):
self.params[key] = handler
def __iter__(self):
return self.params.__iter__()
def __len__(self):
return len(self.params)
def __getitem__(self, key):
return self.params[key].read(self.content)
def __setitem__(self, key, value):
self.content = self.params[key].write(self.content, value)
def keys(self):
return self.params.keys()
def __delitem__(self, key):
del self.params[key]
class IndexedDict(Mapping):
"""Wrapper around OrderedDict that allows access via item position or key"""
def __init__(self, *args, **kwargs):
self._od = OrderedDict(*args, **kwargs)
def __getitem__(self, k):
try: return list(self._od.values())[k]
except TypeError: return self._od[k]
def __len__(self): return self._od.__len__()
def __iter__(self): return self._od.__iter__()
class DictContainer(MutableMapping):
"""
Wrapper around a dict, to track the elements' parents.
**This class shouldn't be instantiated directly by users,
but by the elements that contain it**.
:param args: elements contained in the dict--like object
:param oktypes: type or tuple of types that are allowed as items
:type oktypes: ``type`` | ``tuple``
:param parent: the parent element
:type parent: ``Element``
"""
__slots__ = ['dict', 'oktypes', 'parent', 'location']
def __init__(self, *args, oktypes=object, parent=None, **kwargs):
self.oktypes = oktypes
self.parent = parent
self.location = None
self.dict = OrderedDict()
self.update(args) # Must be a sequence of tuples
self.update(kwargs) # Order of kwargs is not preserved
def __contains__(self, item):
return item in self.dict
def __len__(self):
return len(self.dict)
def __getitem__(self, k):
return attach(self.dict[k], self.parent, self.location)
def __delitem__(self, k):
del self.dict[k]
def __setitem__(self, k, v):
v = check_type(v, self.oktypes)
self.dict[k] = v
def __str__(self):
return self.__repr__()
def __repr__(self):
return 'DictContainer({})'.format(' '.join(repr(x) for x in self.dict))
def __iter__(self):
return self.dict.__iter__()
def to_json(self):
items = self.dict.items()
return OrderedDict((k, to_json_wrapper(v)) for k, v in items)
return [item.to_json() for item in self.dict]
class IndexedDict(Mapping):
"""Wrapper around OrderedDict that allows access via item position or key"""
def __init__(self, *args, **kwargs):
self._od = OrderedDict(*args, **kwargs)
def __getitem__(self, k):
try: return list(self._od.values())[k]
except TypeError: return self._od[k]
def __len__(self): return self._od.__len__()
def __iter__(self): return self._od.__iter__()
class MetafeatureFunctions(object):
def __init__(self):
self.functions = OrderedDict()
self.dependencies = OrderedDict()
self.values = OrderedDict()
def clear(self):
self.values = OrderedDict()
def __iter__(self):
return self.functions.__iter__()
def __getitem__(self, item):
return self.functions.__getitem__(item)
def __setitem__(self, key, value):
return self.functions.__setitem__(key, value)
def __delitem__(self, key):
return self.functions.__delitem__(key)
def __contains__(self, item):
return self.functions.__contains__(item)
def get_value(self, key):
return self.values[key].value
def set_value(self, key, item):
self.values[key] = item
def is_calculated(self, key):
"""Return if a helper function has already been executed.
Necessary as get_value() can return None if the helper function hasn't
been executed or if it returned None."""
return key in self.values
def get_dependency(self, name):
"""Return the dependency of metafeature "name".
"""
return self.dependencies.get(name)
def define(self, name, dependency=None):
"""Decorator for adding metafeature functions to a "dictionary" of
metafeatures. This behaves like a function decorating a function,
not a class decorating a function"""
def wrapper(metafeature_class):
instance = metafeature_class()
self.__setitem__(name, instance)
self.dependencies[name] = dependency
return instance
return wrapper
def test_good_OrderedDict_t(self):
"""This tests OrderedDict types"""
a = OrderedDict()
for i, k in zip('abcdef', 'ghijk'):
a.update({i: k})
known_good = (
(a, f_OrderedDict_t, (a,), dict()),
(a, f_OrderedDict_t, (), dict(a=a)),
# Confirm a is the first key (dict give 'd' first)
# f_str takes (a, b='default') so result should be adefault
('adefault', f_str, ((a.__iter__().__next__()),), dict())
)
assert_pass(self, known_good)
class LanguageFamily:
"""
Data class holding the information about a language family, including its name, the names of its known languages
and its master alphabet. A master alphabet is a hypothetical alphabet that is created by taking the union of the
alpabets of the languages in this family.
"""
def __init__(self, language_family_id, name):
"""Protected method. Only to be used by LanngugeCollection."""
self.id = language_family_id
self.name = name
self.languages = OrderedDict()
self._id_to_name = OrderedDict(
) #dictionary instead of list to support stable removal
self._master_alphabet = None
def __len__(self):
return len(self.languages)
def __iter__(self):
return self.languages.__iter__()
def __getitem__(self, index):
if isinstance(index, int):
return self.languages[self._id_to_name[index]]
return self.languages[index]
return self.languages[index]
def add_language(self, language):
"""Protected method. Only to be used by LanguageCollection class."""
self.languages[language.name] = language
self._id_to_name[language.id] = language.name
language.family_id = self.id
self._master_alphabet = None
def get_master_alphabet(self):
"""
Lazily constructs and returns the master alphabet of this family.
:return: An Alphabet object.
"""
if self._master_alphabet is None:
alphabets = [
language.alphabet for _, language in self.languages.items()
]
self._master_alphabet = get_master_alphabet(alphabets,
reindex=True)
return self._master_alphabet
@property
def alphabet(self):
return self.get_master_alphabet()
class BuerFilter(object):
""" 存在的目的是,在短期内,不问服务器同样的问题两次!从而减少网络访问的时间损耗 """
""" 如此我们可以利用一个有序字典! 顺序尾部是最近使用的,顺序头部是很久没用的 """
def __init__(self, contain=2000):
self.filter = OrderedDict() # 去重器
self.contain = contain # 约束去重量
def existent(self, url):
""" 如果 url 存在返回 True 不存在返回 False """
if len(self.filter) >= self.contain and self.filter: # 控制长度
self.filter.pop(next(self.filter.__iter__())) # 踢掉最不常用的
result = self.filter.pop(url, False)
self.filter[url] = True
return result
class ResultStore(object):
"""
Result store interface
"""
def __init__(self):
"""
Initialize a new :py:class:`ResultStore`
"""
self.db = OrderedDict()
def __contains__(self, k):
"""
Check whether the given key is in the :py:class:`ResultStore`
Args:
k (str): key
Returns:
bool: whether or not k is in ``self.db``
"""
return k in self.db
def __iter__(self):
"""
Get an iterable over the keys in ``self.db``
Returns:
iterable: an iterable over the keys in ``self.db``
"""
return self.db.__iter__()
def __setitem__(self, k, v):
"""
Set a (key, value) pair in ``self.db``
Args:
k (str): key
v (str): value
"""
return self.db.__setitem__(k, v)
def itervalues(self):
"""
Get an iterable over the values in ``self.db``
Returns:
iterable: an iterable over the values in ``self.db``
"""
return self.db.itervalues()
class OrderedSet(Set):
def __init__(self):
self.data = OrderedDict()
def add(self, element):
self.data[element] = 0
def __len__(self):
return self.data.__len__()
def __contains__(self, value):
return value in self.data
def __iter__(self):
return self.data.__iter__()
class HelperFunctions:
def __init__(self):
self.functions = OrderedDict()
self.values = OrderedDict()
def clear(self):
self.values = OrderedDict()
self.computation_time = OrderedDict()
def __iter__(self):
return self.functions.__iter__()
def __getitem__(self, item):
return self.functions.__getitem__(item)
def __setitem__(self, key, value):
return self.functions.__setitem__(key, value)
def __delitem__(self, key):
return self.functions.__delitem__(key)
def __contains__(self, item):
return self.functions.__contains__(item)
def is_calculated(self, key):
"""Return if a helper function has already been executed.
Necessary as get_value() can return None if the helper function hasn't
been executed or if it returned None."""
return key in self.values
def get_value(self, key):
return self.values.get(key).value
def set_value(self, key, item):
self.values[key] = item
def define(self, name):
"""Decorator for adding helper functions to a "dictionary".
This behaves like a function decorating a function,
not a class decorating a function"""
def wrapper(metafeature_class):
instance = metafeature_class()
self.__setitem__(name, instance)
return instance
return wrapper
class Answers(object):
def __init__(self, raw_answers, aid_to_answer_common):
self.aid_to_answer = OrderedDict()
for raw_answer in raw_answers:
aid = raw_answer["id"]
self[aid] = Answer(raw_answer)
aid_to_answer_common.update(self.aid_to_answer)
def __setitem__(self, key, value):
self.aid_to_answer[key] = value
def __iter__(self):
return self.aid_to_answer.__iter__()
def get_all(self):
return self.aid_to_answer.values()
class TableFrame(InfoFrame):
def __init__(self, *args, **kwargs):
InfoFrame.__init__(self, *args, **kwargs)
self.headings = OrderedDict()
self.items = OrderedDict()
def set_column_headings(self, *headings, **kwargs):
self.headings = OrderedDict()
self.items = OrderedDict()
col = 0
for h in headings:
self.headings[h] = col
TK.Label(self, text=h, **kwargs).grid(row=0, column=col)
col += 1
def add_row(self, iid, **kwargs):
self.items[iid] = {}
col = 0
for h in self.headings:
self.items[iid][h] = TK.StringVar()
if "bg" not in kwargs and "background" not in kwargs:
kwargs["bg"] = self.config("bg")[4]
if "fg" not in kwargs and "foreground" not in kwargs:
kwargs["fg"] = self.config("fg")[4]
TK.Label(self, textvariable=self.items[iid][h],
**kwargs).grid(row=len(self.items), column=col)
col += 1
def set_variable(self, iid, heading, value):
self.items[iid][heading].set(value)
def get_variable(self, iid, heading):
return self.items[iid][heading].get()
def __contains__(self, iid):
return iid in self.items
def __iter__(self):
return self.items.__iter__()
def __getitem__(self, iid):
return self.items[iid]
def set_row(self, iid, **kwargs):
for k in kwargs:
self.set_variable(iid, k, kwargs[k])
class InstanceCollection():
INSTANCES = None
def __init__(self):
self.INSTANCES = OrderedDict()
def add_instance(self, ec2instance):
if not isinstance(ec2instance, Instance):
return False
self.INSTANCES[len(self.INSTANCES)] = ec2instance
return self
def __str__(self):
columns = ['UID','IID','Name','Type','Public DNS','AZ','AMI','State']
#columns.append()
#("UID", "IID", "Name", "Type", "ELB", "AZ", "Public DNS", "Ami", "State", "CPU")
rows = []
for c, i in self.INSTANCES.items():
row_data = [c, i.id, i.tags.get("Name", ""), i.instance_type, i.dns_name, i.placement, i.image_id, i._state]
if 'load_balancer' in i.__dict__:
if 'ELB' not in columns:
columns.append('ELB')
row_data.append(i.load_balancer)
if 'cpu_utilization' in self.INSTANCES[0].__dict__:
if 'CPU' not in columns:
columns.append('CPU')
row_data.append(i.cpu_utilization)
rows.append(tuple(row_data))
table = Table(columns)
for row in rows:
table.add_row(row)
return str(table)
def __len__(self):
return len(self.INSTANCES)
def __iter__(self):
return self.INSTANCES.__iter__()
def __getitem__(self, index):
return self.INSTANCES[index]
class OrderedSet:
def __init__(self, items=None):
self.d = OrderedDict()
self.update(items)
def update(self, items):
if items is not None:
for item in items:
self.d[item] = 1
def __iter__(self):
return self.d.__iter__()
def __contains__(self, key):
return self.d.__contains__(key)
def __delitem__(self, key):
return self.d.__delitem__(key)
def __len__(self):
return self.d.__len__()
def add(self, x):
return update(self, [x])
def discard(self, x):
if self.__contains__(x):
return self.__del__(x)
def remove(self, x):
if not self.__contains__(x):
raise KeyError
return self.__del__(x)
def _format_op(self, op):
if hasattr(op, 'name'):
return op.name
return str(op)
def __repr__(self):
if not self:
return '%s()' % (self.__class__.__name__,)
return '{%r}' % (','.join([self._format_op(op) for op in self]),)
class enum(object):
def __init__(self, *sequential, **named):
self.__enums = OrderedDict(zip(sequential, range(len(sequential))), **named)
for key, val in self.__enums.items():
setattr(self, key, val)
def __getitem__(self, key):
return self.__enums[key]
def items(self):
return self.__enums.items()
def keys(self):
return self.__enums.keys()
def values(self):
return self.__enums.values()
def __iter__(self):
return self.__enums.__iter__()
class OrderedDefaultDict(object):
"""
Implements (most of) a default dict with ordering.
"""
def __init__(self, factory):
self._factory = factory
self._dict = OrderedDict()
def __setitem__(self, key, item):
self._dict.__setitem__(key, item)
def __getitem__(self, key):
if key not in self._dict:
self._dict[key] = self._factory()
return self._dict.__getitem__(key)
def __repr__(self):
return self._dict.__repr__()
def __iter__(self):
return self._dict.__iter__()
class WeightedRegistry(Registry):
"""A registry that returns registered classes in
weight order. The classes that are registered should have
a 'weight' attribute.
"""
def __init__(self, *args, **kwargs):
self._ordered_dict = OrderedDict()
super(WeightedRegistry, self).__init__(*args, **kwargs)
def register(self, klass):
super(WeightedRegistry, self).register(klass)
key = self._get_key_from_class(klass)
self._ordered_dict[key] = klass
self._ordered_dict = OrderedDict(
sorted(self._ordered_dict.iteritems(),
key=lambda item: item[1].weight))
def __iter__(self):
return self._ordered_dict.__iter__()
def values(self):
return self._ordered_dict.values()
class QuestionTable(object):
def __init__(self):
self._table = OrderedDict()
def __iter__(self):
return self._table.__iter__()
def questions(self):
return [q for qList in self._table.values() for q in qList]
def identifiers(self):
return self._table.keys()
def add(self, question):
questionList = self._table.get(question.identifier, QuestionList())
questionList.append(question)
self._table[question.identifier] = questionList
def get(self, identifier):
questions = self._table.get(identifier, None)
if questions:
return questions.getVisibleQuestion()
def getType(self, identifier):
questions = self._table.get(identifier, None)
if questions:
return questions[0].type
def getQuestionList(self, identifier):
return self._table.get(identifier, None)
class ResourceDirectory(Resource):
def __init__(self, display_string, selector, host, port, extra):
super().__init__(
RESOURCE_TYPES['directory'], display_string, selector, host, port,
)
self._resources = OrderedDict()
def __iter__(self):
return self._resources.__iter__()
def __getitem__(self, item):
return self._resources[item]
def add(self, item):
self._resources[item.selector] = item
def items(self):
return self._resources.items()
def display(self):
for resource in self._resources.values():
yield resource.as_menu_item().encode("utf-8") + utils.EOF
yield b'.' + utils.EOF
class DotMap(OrderedDict):
def __init__(self, *args, **kwargs):
self._map = OrderedDict()
self._dynamic = True
if kwargs:
if '_dynamic' in kwargs:
self._dynamic = kwargs['_dynamic']
if args:
d = args[0]
if isinstance(d, dict):
for k,v in self.__call_items(d):
if type(v) is dict:
v = DotMap(v, _dynamic=self._dynamic)
if type(v) is list:
l = []
for i in v:
n = i
if type(i) is dict:
n = DotMap(i, _dynamic=self._dynamic)
l.append(n)
v = l
self._map[k] = v
if kwargs:
for k,v in self.__call_items(kwargs):
if k is not '_dynamic':
self._map[k] = v
def __call_items(self, obj):
if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
return obj.iteritems()
else:
return obj.items()
def items(self):
return self.iteritems()
def iteritems(self):
return self.__call_items(self._map)
def __iter__(self):
return self._map.__iter__()
def next(self):
return self._map.next()
def __setitem__(self, k, v):
self._map[k] = v
def __getitem__(self, k):
if k not in self._map and self._dynamic and k != '_ipython_canary_method_should_not_exist_':
# automatically extend to new DotMap
self[k] = DotMap()
return self._map[k]
def __setattr__(self, k, v):
if k in {'_map','_dynamic', '_ipython_canary_method_should_not_exist_'}:
super(DotMap, self).__setattr__(k,v)
else:
self[k] = v
def __getattr__(self, k):
if k == {'_map','_dynamic','_ipython_canary_method_should_not_exist_'}:
super(DotMap, self).__getattr__(k)
else:
return self[k]
def __delattr__(self, key):
return self._map.__delitem__(key)
def __contains__(self, k):
return self._map.__contains__(k)
def __str__(self):
items = []
for k,v in self.__call_items(self._map):
# bizarre recursive assignment situation (why someone would do this is beyond me)
if id(v) == id(self):
items.append('{0}=DotMap(...)'.format(k))
else:
items.append('{0}={1}'.format(k, repr(v)))
out = 'DotMap({0})'.format(', '.join(items))
return out
def __repr__(self):
return str(self)
def toDict(self):
d = {}
for k,v in self.items():
if type(v) is DotMap:
# bizarre recursive assignment support
if id(v) == id(self):
v = d
else:
v = v.toDict()
elif type(v) is list:
l = []
for i in v:
n = i
if type(i) is DotMap:
n = i.toDict()
l.append(n)
v = l
d[k] = v
return d
def pprint(self):
pprint(self.toDict())
def empty(self):
return (not any(self))
# proper dict subclassing
def values(self):
return self._map.values()
# ipython support
def __dir__(self):
return self.keys()
@classmethod
def parseOther(self, other):
if type(other) is DotMap:
return other._map
else:
return other
def __cmp__(self, other):
other = DotMap.parseOther(other)
return self._map.__cmp__(other)
def __eq__(self, other):
other = DotMap.parseOther(other)
if not isinstance(other, dict):
return False
return self._map.__eq__(other)
def __ge__(self, other):
other = DotMap.parseOther(other)
return self._map.__ge__(other)
def __gt__(self, other):
other = DotMap.parseOther(other)
return self._map.__gt__(other)
def __le__(self, other):
other = DotMap.parseOther(other)
return self._map.__le__(other)
def __lt__(self, other):
other = DotMap.parseOther(other)
return self._map.__lt__(other)
def __ne__(self, other):
other = DotMap.parseOther(other)
return self._map.__ne__(other)
def __delitem__(self, key):
return self._map.__delitem__(key)
def __len__(self):
return self._map.__len__()
def clear(self):
self._map.clear()
def copy(self):
return DotMap(self.toDict())
def get(self, key, default=None):
return self._map.get(key, default)
def has_key(self, key):
return key in self._map
def iterkeys(self):
return self._map.iterkeys()
def itervalues(self):
return self._map.itervalues()
def keys(self):
return self._map.keys()
def pop(self, key, default=None):
return self._map.pop(key, default)
def popitem(self):
return self._map.popitem()
def setdefault(self, key, default=None):
self._map.setdefault(key, default)
def update(self, *args, **kwargs):
if len(args) != 0:
self._map.update(*args)
self._map.update(kwargs)
def viewitems(self):
return self._map.viewitems()
def viewkeys(self):
return self._map.viewkeys()
def viewvalues(self):
return self._map.viewvalues()
@classmethod
def fromkeys(cls, seq, value=None):
d = DotMap()
d._map = OrderedDict.fromkeys(seq, value)
return d
def __getstate__(self): return self.__dict__
def __setstate__(self, d): self.__dict__.update(d)
class ConfigSpace(object):
"""The configuration space of a schedule. Pass it as config in template to
collect transformation space and build transform graph of axes
"""
def __init__(self):
# private dict to provide sugar
self.space_map = OrderedDict() # name -> space
self._collect = True
self._length = None
self._entity_map = OrderedDict() # name -> entity
self._constraints = []
self.errors = []
self.template_key = None
self.code_hash = None
self.flop = 0
self.is_fallback = False
@staticmethod
def axis(var):
"""get a virtual axis (axis placeholder)
Parameters
----------
var: int or tvm.schedule.IterVar
If is int, return an axis whose length is the provided argument.
If is IterVar, return an axis whose length is extracted from the
IterVar's extent domain.
"""
return VirtualAxis(var)
reduce_axis = axis
def define_split(self, name, axis, policy='all', **kwargs):
"""Define a new tunable knob which splits an axis into a list of axes
Parameters
----------
name: str
name to index the entity of this space
axis: tvm.schedule.IterVar
axis to split
policy: str
name of policy.
If is 'all', the tuner will try all divisible factors.
If is 'candidate', try listed candidate.
kwargs: dict
extra arguments for policy
see examples below for how to use filter
Examples
--------
>>> # use custom candidates
>>> cfg.define_split('tile_x', x, policy='candidate', candidate=[[1, 4, 4], [4, 1, 4]])
>>> # use a filter that only accepts the split scheme whose inner most tile is less then 4
>>> cfg.define_split('tile_y', y, policy='all', filter=lambda x: x.size[-1] <= 4)
"""
axes = [axis]
return self._add_new_transform(SplitSpace, name, axes, policy, **kwargs)
def define_reorder(self, name, axes, policy, **kwargs):
"""Define a new tunable knob which reorders a list of axes
Parameters
----------
name: str
name to index the entity of this space
axes: Array of tvm.schedule.IterVar
axes to reorder
policy: str
name of policy
If is 'identity', do an identity permutation.
If is 'all', try all permutations.
If is 'interval_all', try all permutations of an interval of axes.
If is 'candidate', try listed candidate.
If is 'interleave', interleave chains of spatial axes and chains of reduction axes.
kwargs: dict
extra arguments for policy
"""
return self._add_new_transform(ReorderSpace, name, axes, policy, **kwargs)
def define_annotate(self, name, axes, policy, **kwargs):
"""Define a new tunable knob which annotates a list of axes
Parameters
----------
name: str
name to index the entity of this space
axes: Array of tvm.schedule.IterVar
axes to annotate
policy: str
name of policy
If is 'unroll', unroll the axes.
If is 'try_unroll', try to unroll the axes.
If is 'try_unroll_vec', try to unroll or vectorize the axes.
If is 'bind_gpu', bind the first few axes to gpu threads.
If is 'locate_cache', choose n axes to attach shared/local cache.
kwargs: dict
extra arguments for policy
"""
return self._add_new_transform(AnnotateSpace, name, axes, policy, **kwargs)
def define_knob(self, name, candidate):
"""Define a tunable knob with a list of candidates
Parameters
----------
name: str
name key of that option
candidate: list
list of candidates
"""
return self._add_new_transform(OtherOptionSpace, name, [], None, candidate=candidate)
def add_flop(self, flop):
"""Add float operation statistics for this tuning task
Parameters
---------
flop: int or float
number of float operations
"""
self.flop += flop
def raise_error(self, msg):
"""register error in config
Using this to actively detect error when scheudling.
Otherwise these error will occur during runtime, which
will cost more time.
Parameters
----------
msg: str
"""
self.errors.append(msg)
def valid(self):
"""Check whether the config meets all the constraints
Note: This check should be called after instantiation of task,
because the ConfigEntity/ConfigSpace collects errors during instantiation
Returns
-------
valid: bool
whether the config meets all the constraints
"""
return not bool(self.errors)
def _add_new_transform(self, space_class, name, axes, policy, **kwargs):
"""Add a new transform space in template"""
if self._collect:
# convert schedule axis to space definition axis
axes = [x if isinstance(x, (VirtualAxis, Axis)) else self.axis(x) for x in axes]
# add subspace (knob)
space = space_class(axes, policy, **kwargs)
self.space_map[name] = space
self._entity_map[name] = space[0]
return [Axis(space, i) for i in range(space.num_output)]
return [Axis(None, i) for i in range(space_class.get_num_output(axes, policy, **kwargs))]
def __len__(self):
if self._length is None:
self._length = int(np.prod([len(x) for x in self.space_map.values()]))
return self._length
def get(self, index):
"""Get a config entity with detailed parameters from this space
Parameters
----------
index: int
index in the space
"""
entities = OrderedDict()
t = index
for name, space in self.space_map.items():
entities[name] = space[t % len(space)]
t //= len(space)
ret = ConfigEntity(index, self.code_hash, self.template_key, entities, self._constraints)
return ret
def __iter__(self):
return self._entity_map.__iter__()
def __getitem__(self, name):
"""get the transform entity(knob) of this entity by name
do not use this to get a ConfigEntity of this space (should use ConfigSpace.get instead)
Parameters
----------
name: str
name of the transform
"""
return self._entity_map[name]
def __repr__(self):
res = "ConfigSpace (len=%d, space_map=\n" % len(self)
for i, (name, space) in enumerate(self.space_map.items()):
res += " %2d %s: %s\n" % (i, name, space)
return res + ")"
class VecWrapper(object):
"""
A dict-like container of a collection of variables.
Args
----
pathname : str, optional
the pathname of the containing `System`
comm : an MPI communicator (real or fake)
a communicator that can be used for distributed operations
when running under MPI. If not running under MPI, it is
ignored
Attributes
----------
idx_arr_type : dtype, optional
A dtype indicating how index arrays are to be represented.
The value 'i' indicates an numpy integer array, other
implementations, e.g., petsc, will define this differently.
"""
idx_arr_type = 'i'
def __init__(self, sysdata, comm=None):
self.comm = comm
self.vec = None
self._dat = OrderedDict()
self._has_pbos = False
# Automatic unit conversion in target vectors
self.deriv_units = False
self._sysdata = sysdata
def _flat(self, name):
"""
Return a flat version of the named variable, including any necessary conversions.
"""
return self._dat[name].flat()
def metadata(self, name):
"""
Returns the metadata for the named variable.
Args
----
name : str
Name of variable to get the metadata for.
Returns
-------
dict
The metadata dict for the named variable.
Raises
-------
KeyError
If the named variable is not in this vector.
"""
try:
return self._dat[name].meta
except KeyError as error:
msg = "Variable '{name}' does not exist".format(name=name)
raise KeyError(msg)
def __getitem__(self, name):
"""
Retrieve unflattened value of named var.
Args
----
name : str
Name of variable to get the value for.
Returns
-------
The unflattened value of the named variable.
"""
return self._dat[name].get()
def __setitem__(self, name, value):
"""
Set the value of the named variable.
Args
----
name : str
Name of variable to get the value for.
value :
The unflattened value of the named variable.
"""
self._dat[name].set(value)
def __len__(self):
"""
Returns
-------
The number of keys (variables) in this vector.
"""
return len(self._dat)
def __contains__(self, key):
"""
Returns
-------
A boolean indicating if the given key (variable name) is in this vector.
"""
return key in self._dat
def __iter__(self):
"""
Returns
-------
A dictionary iterator over the items in _dat.
"""
return self._dat.__iter__()
def veciter(self):
"""
Returns
-------
An iterator over names and values of all variables found in the
flattened vector, i.e., no pass_by_obj variables.
"""
if self._has_pbos:
return ((n, acc.val) for n, acc in iteritems(self._dat)
if not acc.meta.get('pass_by_obj'))
else:
return ((n, acc.val) for n, acc in iteritems(self._dat))
def keys(self):
"""
Returns
-------
list or KeyView (python 3)
the keys (variable names) in this vector.
"""
return self._dat.keys()
def iterkeys(self):
"""
Returns
-------
iter of str
the keys (variable names) in this vector.
"""
return iterkeys(self._dat)
def items(self):
"""
Returns
-------
list of (str, dict)
List of tuples containing the name and metadata dict for each
variable.
"""
return [(name, acc.meta) for name, acc in iteritems(self._dat)]
def iteritems(self):
"""
Returns
-------
iterator
Iterator returning the name and metadata dict for each variable.
"""
return ((name, acc.meta) for name, acc in iteritems(self._dat))
def values(self):
"""
Returns
-------
list of dict
List containing metadata dict for each variable.
"""
return [acc.meta for acc in itervalues(self._dat)]
def itervalues(self):
"""
Returns
-------
iter of dict
Iterator yielding metadata dict for each variable.
"""
return (acc.meta for acc in itervalues(self._dat))
def _get_local_idxs(self, name, idx_dict, get_slice=False):
"""
Returns all of the indices for the named variable in this vector.
Args
----
name : str
Name of variable to get the indices for.
get_slice : bool, optional
If True, return the idxs as a slice object, if possible.
Returns
-------
size
The size of the named variable.
ndarray
Index array containing all local indices for the named variable.
"""
try:
slc = self._dat[name].slice
if slc is None:
return self.make_idx_array(0, 0)
except KeyError:
# this happens if 'name' doesn't exist in this process
return self.make_idx_array(0, 0)
start, end = slc
if name in idx_dict:
#TODO: possible slice conversion
idxs = self.to_idx_array(idx_dict[name]) + start
if idxs.size > (end-start) or max(idxs) >= end:
raise RuntimeError("Indices of interest specified for '%s'"
"are too large" % name)
return idxs
else:
if get_slice:
return slice(start, end)
return self.make_idx_array(start, end)
def norm(self):
"""
Calculates the norm of this vector.
Returns
-------
float
Norm of our internal vector.
"""
return norm(self.vec)
def get_view(self, system, comm, varmap):
"""
Return a new `VecWrapper` that is a view into this one.
Args
----
system : `System`
System for which the view is being created.
comm : an MPI communicator (real or fake)
A communicator that is used in the creation of the view.
varmap : dict
Mapping of variable names in the old `VecWrapper` to the names
they will have in the new `VecWrapper`.
Returns
-------
`VecWrapper`
A new `VecWrapper` that is a view into this one.
"""
view = self.__class__(system._sysdata, comm)
view_size = 0
start = -1
# varmap is ordered, in the same order as _dat
for name, pname in iteritems(varmap):
if name in self._dat:
meta = self._dat[name].meta
pbo = meta.get('pass_by_obj')
if pbo or meta.get('remote'):
view._dat[pname] = Accessor(view, None,
self._dat[name].val, meta)
if pbo:
view._has_pbos = True
else:
pstart, pend = self._dat[name].slice
if start == -1:
start = pstart
end = pend
else:
assert pstart == end, \
"%s not contiguous in block containing %s" % \
(name, varmap.keys())
end = pend
view._dat[pname] = Accessor(view,
(view_size, view_size + meta['size']),
self._dat[name].val, meta)
view_size += meta['size']
if start == -1: # no items found
view.vec = self.vec[0:0]
else:
view.vec = self.vec[start:end]
return view
def make_idx_array(self, start, end):
"""
Return an index vector of the right int type for
the current implementation.
Args
----
start : int
The starting index.
end : int
The ending index.
Returns
-------
ndarray of idx_arr_type
index array containing all indices from start up to but
not including end
"""
return numpy.arange(start, end, dtype=self.idx_arr_type)
def to_idx_array(self, indices):
"""
Given some iterator of indices, return an index array of the
right int type for the current implementation.
Args
----
indices : iterator of ints
An iterator of indices.
Returns
-------
ndarray of idx_arr_type
Index array containing all of the given indices.
"""
return numpy.array(indices, dtype=self.idx_arr_type)
def merge_idxs(self, idxs):
"""
Return source and target index arrays, built up from
smaller index arrays.
Args
----
idxs : array
Indices.
Returns
-------
ndarray of idx_arr_type
Index array containing all of the merged indices.
"""
if len(idxs) == 0:
return self.make_idx_array(0, 0)
return numpy.concatenate(idxs)
def get_states(self):
"""
Returns
-------
list
A list of names of state variables.
"""
return [n for n, meta in iteritems(self) if meta.get('state')]
def _get_vecvars(self):
"""
Returns
-------
A list of names of variables found in our 'vec' array. This includes
params that are not 'owned' and remote vars, which have size 0 array values.
"""
return ((n, meta) for n, meta in self.iteritems()
if not meta.get('pass_by_obj'))
def _scoped_abs_name(self, name):
"""
Args
----
name : str
The absolute pathname of a variable.
Returns
-------
str
The given name as seen from the 'scope' of the `System` that
contains this `VecWrapper`.
"""
if self._sysdata.pathname:
return name[len(self._sysdata.pathname)+1:]
else:
return name
def dump(self, out_stream=sys.stdout): # pragma: no cover
"""
Args
----
out_stream : file_like
Where to send human readable output. Default is sys.stdout. Set to
None to return a str.
"""
if out_stream is None:
out_stream = cStringIO()
return_str = True
else:
return_str = False
lens = [len(n) for n in self.keys()]
nwid = max(lens) if lens else 10
vlens = [len(repr(self[v])) for v in self.keys()]
vwid = max(vlens) if vlens else 1
if self._has_pbos: # we have some pass by obj
defwid = 8
else:
defwid = 1
slens = [len('[{0[0]}:{0[1]}]'.format(self._dat[v].slice))
for v in self.keys()
if self._dat[v].slice is not None]+[defwid]
swid = max(slens)
for v, meta in iteritems(self):
if meta.get('pass_by_obj') or meta.get('remote'):
continue
if self._dat[v].slice is not None:
uslice = '[{0[0]}:{0[1]}]'.format(self._dat[v].slice)
else:
uslice = ''
template = "{0:<{nwid}} {1:<{swid}} {2:>{vwid}}\n"
out_stream.write(template.format(v,
uslice,
repr(self[v]),
nwid=nwid,
swid=swid,
vwid=vwid))
for v, meta in iteritems(self):
if meta.get('pass_by_obj') and not meta.get('remote'):
template = "{0:<{nwid}} {1:<{swid}} {2}\n"
out_stream.write(template.format(v, '(by obj)',
repr(self[v]),
nwid=nwid,
swid=swid))
if return_str:
return out_stream.getvalue()
