class RuntimeContext(object):
INSTANCE = None
def __init__(self):
# shutdown_hook集合，顺序执行
self.__shutdown_hook = OrderedDict()
self._register_signal()
def add_shutdown_hook(self, name, fn):
if isinstance(name, str) and type(fn).__name__ == "function":
self.__shutdown_hook[name] = fn
return {"name": name, "fn": fn}
else:
return None
def remove_shutdown_hook(self, name):
if isinstance(name, str) and name in self.__shutdown_hook:
return self.__shutdown_hook[name]
else:
return None
def __call(self, signum, frame):
for name in self.__shutdown_hook.__reversed__():
if str(name).startswith("$sys_"):
# 执行系统默认的关闭方法
if str(name) == "$sys_#" + str(signal.SIGINT):
self.__shutdown_hook[name](signum, frame)
else:
try:
self.__shutdown_hook[name](signum, frame)
except Exception:
pass
def _register_signal(self):
from piz_base.common.enum import OSTypeEnum
from piz_base.common.tool_utils import SystemUtils
# ctrl+c 信号
self.__shutdown_hook["$sys_#" + str(signal.SIGINT)] = signal.getsignal(
signal.SIGINT)
signal.signal(signal.SIGINT, lambda s, f: self.__call(s, f))
# 非windows信号
if SystemUtils.LOCAL_OS != OSTypeEnum.WINDOWS:
# nohup关闭信号
self.__shutdown_hook["$sys_#" +
str(signal.SIGHUP)] = signal.getsignal(
signal.SIGHUP)
signal.signal(signal.SIGHUP, lambda s, f: self.__call(s, f))
# kill pid信号
self.__shutdown_hook["$sys_#" +
str(signal.SIGTERM)] = signal.getsignal(
signal.SIGTERM)
signal.signal(signal.SIGTERM, lambda s, f: self.__call(s, f))
def elements_by_freq(duplicates):
"""
:param duplicates: a list of string containing duplicates
:return: a list of the set of elements with decreasing frequency
>>> elements_by_freq(['table', 'chair', 'table', 'lamp', 'lamp', 'lamp'])
['lamp', 'table', 'chair']
"""
fd = nltk.FreqDist(duplicates)
dist = OrderedDict(sorted(fd.items(), key=lambda t: t[1]))
dist = dist.__reversed__()
[str(word) for word in dist]
class IndexedGrad:
"""OrderedDict-like object representing a set of cached gradients indexed by
variables wrt to which the grad was taken. Values of gradient are stored in
variables and must be explictly updated using "update" method.
Implements Python dictionary interface delegating to underlying
{var: cached_grad} dict
g = IndexedGrad(vars_=vars_, grads=grads) # create from existing grad tensors
g = IndexedGrad(loss=loss, vars_=vars_) # use tf.gradients to get grad tensors
g.f # concated vector version of all cached grads
g.update() # updates cached value of grads
g.live # live version of gradients [tensor1,tensor2...
g.cached # cached version of grads [var1,var2...
g[param] # cached version of grad for param
# TODO: need better way to get live gradients out
"""
def __init__(self, *forbidden, grads=None, vars_=None, loss=None):
assert len(forbidden) == 0 # force params to be keyword-only
assert vars_ is not None
if grads==None:
assert loss is not None
grads = tf.gradients(loss, vars_)
self.vars_ = vars_
self.grads_dict = OrderedDict()
self.cached = []
self.live = []
update_ops = []
for grad, var in zip(grads, vars_):
self.live.append(grad)
cached = tf.Variable(grad, var.name+"_grad_cached")
self.cached.append(cached)
self.grads_dict[var] = cached
update_ops.append(cached.initializer)
self.update_op = tf.group(*update_ops)
self.f = u.flatten(self.cached)
self.cached = VarList(self.cached)
assert list(self.cached) == list(self.grads_dict.values())
def update(self):
"""Upgrade cached gradients using from their live values using
default session."""
sess = u.get_default_session()
# sess.run(self.update_op)
u.run(self.update_op)
def __len__(self):
return self.grads_dict.__len__()
def __length_hint__(self):
return self.grads_dict.__length_hint__()
def __getitem__(self, key):
return self.grads_dict.__getitem__(key)
def __missing__(self, key):
return self.grads_dict.__missing__(key)
def __iter__(self):
return self.grads_dict.__iter__()
def __reversed__(self):
return self.grads_dict.__reversed__()
def __contains__(self, item):
return self.grads_dict.__contains__(item)
def from_grads_and_vars(grads_and_vars):
grads = [g[0] for g in grads_and_vars]
vars_ = [g[1] for g in grads_and_vars]
return IndexedGrad(grads=grads, vars_=vars_)
def to_grads_and_vars(self):
grads = list(self.cached)
vars_ = list(self.vars_)
return zip(grads, vars_)
class BaseCache(metaclass=ABCMeta):
"""
BaseCache is a class that saves and operates on an OrderedDict. It has a
certain capacity, stored in the attribute `maxsize`. Whether this
capacity is reached, can be checked by using the boolean property
`is_full`. To implement a custom cache, inherit from this class and
override the methods ``__getitem__`` and ``__setitem__``.
Call the method `sunpy.database.caching.BaseCache.callback` as soon
as an item from the cache is removed.
"""
def __init__(self, maxsize=float('inf')):
self.maxsize = maxsize
self._dict = OrderedDict()
def get(self, key, default=None): # pragma: no cover
"""Return the corresponding value to `key` if `key` is in the cache,
`default` otherwise. This method has no side-effects, multiple calls
with the same cache and the same passed key must always return the same
value.
"""
try:
return self._dict[key]
except KeyError:
return default
@abstractmethod
def __getitem__(self, key):
"""abstract method: this method must be overwritten by inheriting
subclasses. It defines what happens if an item from the cache is
attempted to be accessed.
"""
return # pragma: no cover
@abstractmethod
def __setitem__(self, key, value):
"""abstract method: this method must be overwritten by inheriting
subclasses. It defines what happens if a new value should be assigned
to the given key. If the given key does already exist in the cache or
not must be checked by the person who implements this method.
"""
@abstractproperty
def to_be_removed(self):
"""The item that will be removed on the next
:meth:`sunpy.database.caching.BaseCache.remove` call.
"""
@abstractmethod
def remove(self):
"""Call this method to manually remove one item from the cache. Which
item is removed, depends on the implementation of the cache. After the
item has been removed, the callback method is called.
"""
def callback(self, key, value):
"""This method should be called (by convention) if an item is removed
from the cache because it is full. The passed key and value are the
ones that are removed. By default this method does nothing, but it
can be customized in a custom cache that inherits from this base class.
"""
@property
def is_full(self):
"""True if the number of items in the cache equals :attr:`maxsize`,
False otherwise.
"""
return len(self._dict) == self.maxsize
def __delitem__(self, key):
self._dict.__delitem__(key)
def __contains__(self, key):
return key in self._dict.keys()
def __len__(self):
return len(self._dict)
def __iter__(self):
yield from self._dict.__iter__()
def __reversed__(self): # pragma: no cover
yield from self._dict.__reversed__()
def clear(self): # pragma: no cover
return self._dict.clear()
def keys(self): # pragma: no cover
return list(self._dict.keys())
def values(self): # pragma: no cover
return list(self._dict.values())
def items(self): # pragma: no cover
return list(self._dict.items())
def iterkeys(self): # pragma: no cover
return iter(self._dict.keys())
def itervalues(self): # pragma: no cover
yield from self._dict.values()
def iteritems(self): # pragma: no cover
yield from self._dict.items()
def update(self, *args, **kwds): # pragma: no cover
self._dict.update(*args, **kwds)
def pop(self,
key,
default=MutableMapping._MutableMapping__marker
): # pragma: no cover
return self._dict.pop(key, default)
def setdefault(self, key, default=None): # pragma: no cover
return self._dict.setdefault(key, default)
def popitem(self, last=True): # pragma: no cover
return self._dict.popitem(last)
def __reduce__(self): # pragma: no cover
return self._dict.__reduce__()
def copy(self): # pragma: no cover
return self._dict.copy()
def __eq__(self, other): # pragma: no cover
return self._dict.__eq__(other)
def __ne__(self, other): # pragma: no cover
return self._dict.__ne__(other)
def viewkeys(self): # pragma: no cover
return self._dict.keys()
def viewvalues(self): # pragma: no cover
return self._dict.values()
def viewitems(self): # pragma: no cover
return self._dict.items()
@classmethod
def fromkeys(cls, iterable, value=None): # pragma: no cover
return OrderedDict.fromkeys(iterable, value)
def __repr__(self): # pragma: no cover
return '{}({!r})'.format(self.__class__.__name__, dict(self._dict))
def __reversed__(self):
for k in OrderedDict.__reversed__(self):
yield self.__map[k]
class BaseCache(object):
"""
BaseCache is a class that saves and operates on an OrderedDict. It has a
certain capacity, stored in the attribute `maxsize`. Whether this
capacity is reached, can be checked by using the boolean property
`is_full`. To implement a custom cache, inherit from this class and
override the methods ``__getitem__`` and ``__setitem__``.
Call the method `sunpy.database.caching.BaseCache.callback` as soon
as an item from the cache is removed.
"""
def __init__(self, maxsize=float('inf')):
self.maxsize = maxsize
self._dict = OrderedDict()
def get(self, key, default=None): # pragma: no cover
"""Return the corresponding value to `key` if `key` is in the cache,
`default` otherwise. This method has no side-effects, multiple calls
with the same cache and the same passed key must always return the same
value.
"""
try:
return self._dict[key]
except KeyError:
return default
@abstractmethod
def __getitem__(self, key):
"""abstract method: this method must be overwritten by inheriting
subclasses. It defines what happens if an item from the cache is
attempted to be accessed.
"""
return # pragma: no cover
@abstractmethod
def __setitem__(self, key, value):
"""abstract method: this method must be overwritten by inheriting
subclasses. It defines what happens if a new value should be assigned
to the given key. If the given key does already exist in the cache or
not must be checked by the person who implements this method.
"""
@abstractproperty
def to_be_removed(self):
"""The item that will be removed on the next
:meth:`sunpy.database.caching.BaseCache.remove` call.
"""
@abstractmethod
def remove(self):
"""Call this method to manually remove one item from the cache. Which
item is removed, depends on the implementation of the cache. After the
item has been removed, the callback method is called.
"""
def callback(self, key, value):
"""This method should be called (by convention) if an item is removed
from the cache because it is full. The passed key and value are the
ones that are removed. By default this method does nothing, but it
can be customized in a custom cache that inherits from this base class.
"""
@property
def is_full(self):
"""True if the number of items in the cache equals :attr:`maxsize`,
False otherwise.
"""
return len(self._dict) == self.maxsize
def __delitem__(self, key):
self._dict.__delitem__(key)
def __contains__(self, key):
return key in self._dict.keys()
def __len__(self):
return len(self._dict)
def __iter__(self):
for key in self._dict.__iter__():
yield key
def __reversed__(self): # pragma: no cover
for key in self._dict.__reversed__():
yield key
def clear(self): # pragma: no cover
return self._dict.clear()
def keys(self): # pragma: no cover
return list(self._dict.keys())
def values(self): # pragma: no cover
return list(self._dict.values())
def items(self): # pragma: no cover
return list(self._dict.items())
def iterkeys(self): # pragma: no cover
return iter(self._dict.keys())
def itervalues(self): # pragma: no cover
for value in self._dict.values():
yield value
def iteritems(self): # pragma: no cover
for key, value in six.iteritems(self._dict):
yield key, value
def update(self, *args, **kwds): # pragma: no cover
self._dict.update(*args, **kwds)
def pop(self, key, default=MutableMapping._MutableMapping__marker): # pragma: no cover
return self._dict.pop(key, default)
def setdefault(self, key, default=None): # pragma: no cover
return self._dict.setdefault(key, default)
def popitem(self, last=True): # pragma: no cover
return self._dict.popitem(last)
def __reduce__(self): # pragma: no cover
return self._dict.__reduce__()
def copy(self): # pragma: no cover
return self._dict.copy()
def __eq__(self, other): # pragma: no cover
return self._dict.__eq__(other)
def __ne__(self, other): # pragma: no cover
return self._dict.__ne__(other)
def viewkeys(self): # pragma: no cover
return self._dict.keys()
def viewvalues(self): # pragma: no cover
return self._dict.values()
def viewitems(self): # pragma: no cover
return self._dict.items()
@classmethod
def fromkeys(cls, iterable, value=None): # pragma: no cover
return OrderedDict.fromkeys(iterable, value)
def __repr__(self): # pragma: no cover
return '{0}({1!r})'.format(self.__class__.__name__, dict(self._dict))
class Stack(object):
""" Base class to store pandas objects, with special operations to
return as 3d data (eg panel) and to apply functions itemwise. Items are
stored in an ordered dict."""
itemlabel = 'Item'
_magic=['__len__',
'__iter__',
'__reversed__',
'__contains__',
]
def __init__(self, data, keys=None, name='', sort_items=False):
self.name = name
# Dictionary input
if isinstance(data, dict):
logger.debug('Initializing "%s" from dictionary.' % self.full_name)
if sort_items:
logger.debug('Sorting keys')
self._data=OrderedDict(sorted(data.keys(), key=lambda t: t[0]))
else:
self._data=OrderedDict(data)
else:
if not isinstance(data, Iterable):
logger.info('%s constructed from non-iterable... converting '
'data to an iterable' % self.full_name)
data=[data]
if keys:
if not isinstance(keys, Iterable):
logger.info('%s constructed from non-iterable... converting '
'keys to an iterable' % self.full_name)
keys = [keys]
if len(keys) != len(data):
raise ValueError('Length mistmatch: keys and data (%s,%s)'\
% (len(keys), len(data)))
# If keys not passed, generate them
else:
# Zipped data ((key, df), (key, df))
try:
keys, data = zip(*data)
except Exception:
keys=self._gen_keys(len(data))
if len(keys) > 1:
logger.warn("Generating keys %s-%s" % (keys[0], keys[-1]))
else:
logger.warn("Generating key %s" % keys[0])
self._data=OrderedDict( [ (key, data[i]) for (i, key)
in enumerate(keys) ])
@property
def _address(self):
""" Property to make easily accesible by multicanvas """
return mem_address(super(Stack, self).__repr__())
def _gen_keys(self, length):
""" Return a list of itemlables (item0, item1 etc...) using
self.itemlabel and a length"""
logger.debug('Items not found on %s: generating item list' % self.full_name)
return [self.itemlabel+str(i) for i in range(length)]
# --------------------
# Dictionary Interface
def __getitem__(self, keyslice):
""" If single name, used dict interface. If slice or integer, uses
list interface. All results parameterized to key, data pairs, passed
directly into a new Stack.
"""
# Slice as list of strings or int [0, 'foo', 2, 'bar']
if hasattr(keyslice, '__iter__'):
tuples_out = []
for item in keyslice:
if isinstance(item, str):
item = self._data.keys().index(item)
tuples_out.append(self._data.items()[item])
else:
if isinstance(keyslice, int) or isinstance(keyslice, slice):
tuples_out = self._data.items()[keyslice]
else:
tuples_out = [(keyslice, self._data[keyslice])] #keyslice is name
# If single item, return TimeSpectra, else, return new Stack
# Canonical slicing implementaiton; don't change unless good reason
# Because len() wonky with nested tuples (eg (x,y) and [(x1,y1),(x2,y2)]
# are both length two, this will work:
if sum(1 for x in tuples_out) == 2:
return tuples_out[1] #Return timespectra
else:
return self.__class__(tuples_out)
def __delitem__(self, keyslice):
""" Delete a single name, or a keyslice from names/canvas """
if isinstance(keyslice, str):
idx = self.names.index(keyslice)
self.pop(idx)
else:
raise NotImplementedError("Deletion only supports single entry")
def __setitem__(self, name, canvas):
""" """
if name in self.names:
idx = self.names.index(name)
self.pop(idx)
self.insert(idx, name, canvas)
else:
self.names.append(name)
def __getattr__(self, attr):
""" If attribute not found, try attribute lookup in dictionary. If
that is not found, try finding attribute on self._data.
For example, self.keys() will first look for self['keys']. Since
this isn't found, it calls self._data.keys(). But if I do
self.Item1, then it returns self['Item1']. The very rare conflict
case that a user has named the items a method that may already exist
in the dictionary (eg items=['a','b','keys'] is addressed.
"""
if attr in self._data.keys():
if hasattr(self._data, attr):
raise AttributeError('"%s attribute" found in both the items\
and as a method of the underlying dictionary object.'%(attr))
else:
return self[attr]
return getattr(self._data, attr)
# Attributes deferred to self.data /dictionary
def __len__(self):
return self._data.__len__()
def __iter__(self):
return self._data.__iter__()
def __reversed__(self):
return self._data.__reversed__()
def __contains__(self):
return self._data.__contains__()
def as_3d(self):
""" Return 3d structure of data. Default is panel."""
raise Panel(data=self._data)
### Data types without labels
#Is this realy necessary? See pyparty.ParticleManger for possibly more consistent implementation
def get_all(self, attr, astype=tuple):
"""Generator/tuple etc.. of (item, attribute) pairs. """
return put._parse_generator(
((item[0], getattr(item[1], attr)) for item in self.items()), astype)
def _get_unique(self, attr):
""" Inspects Stack itemwise for an attribute for unique values.
If non-unique value for the attributes are found, returns
"mixed".
"""
unique = set(self.get_all(attr, astype=dict).values())
if len(unique) > 1:
return 'mixed'
else:
return tuple(unique)[0] #set doesn't support indexing
def set_all(self, attr, val, inplace=False):
""" Set attributes itemwise.
If not inplace, returns new instance of self"""
if inplace:
for (key, item) in self.items():
try:
setattr(item, attr, val)
except Exception as E:
raise Exception('Could not set %s in "%s". Received the following \
exception:\n "%s"'%(attr, key, E))
else:
out=deepcopy(self._data) #DEEPCOPY
for item in out:
setattr(out[item], attr, val)
return self.__class__(out)
def apply(self, func, *args, **kwargs):
""" Applies a user-passed function, or calls an instance method itemwise.
Parameters:
-----------
func: str or function
If string, must correspond to a method on the object stored
itemwise in the stack. If a function, appliked itemwise to
objects stored.
inplace: False
Special kwarg. If true, self._data modified inplace,
otherwise new specstack is returned.
*args, **kwargs:
func arguments.
Returns:
--------
If not inplace, returns SpecStack after itemwise application.
"""
inplace=kwargs.pop('inplace', False)
if isinstance(func, basestring):
if inplace:
for item in self:
self[item] = getattr(self[item], func)(*args, **kwargs)
else:
return self.__class__(OrderedDict([(k, getattr(v, func)(*args, \
**kwargs)) for k,v in self.items()]))
# function, numpyfunction etc...
else:
if inplace:
for item in self:
self[item] = self[item].apply(func)(*args, **kwargs)
else:
return self.__class__(OrderedDict([(k, v.apply(func, *args, \
**kwargs)) for k,v in self.items()]))
@property
def full_name(self):
""" Timespectra:name or Timespectra:unnamed. Useful for scripts mostly """
outname = getattr(self, 'name', 'unnamed')
return '%s:%s' % (self.__class__.__name__, self.name)
class Stack(object):
""" Base class to store pandas objects, with special operations to
return as 3d data (eg panel) and to apply functions itemwise. Items are
stored in an ordered dict."""
itemlabel = 'Item'
_magic = [
'__len__',
'__iter__',
'__reversed__',
'__contains__',
]
def __init__(self, data, keys=None, name='', sort_items=False):
self.name = name
# Dictionary input
if isinstance(data, dict):
logger.debug('Initializing "%s" from dictionary.' % self.full_name)
if sort_items:
logger.debug('Sorting keys')
self._data = OrderedDict(
sorted(data.keys(), key=lambda t: t[0]))
else:
self._data = OrderedDict(data)
else:
if not isinstance(data, Iterable):
logger.info('%s constructed from non-iterable... converting '
'data to an iterable' % self.full_name)
data = [data]
if keys:
if not isinstance(keys, Iterable):
logger.info(
'%s constructed from non-iterable... converting '
'keys to an iterable' % self.full_name)
keys = [keys]
if len(keys) != len(data):
raise ValueError('Length mistmatch: keys and data (%s,%s)'\
% (len(keys), len(data)))
# If keys not passed, generate them
else:
# Zipped data ((key, df), (key, df))
try:
keys, data = zip(*data)
except Exception:
keys = self._gen_keys(len(data))
if len(keys) > 1:
logger.warn("Generating keys %s-%s" %
(keys[0], keys[-1]))
else:
logger.warn("Generating key %s" % keys[0])
self._data = OrderedDict([(key, data[i])
for (i, key) in enumerate(keys)])
@property
def _address(self):
""" Property to make easily accesible by multicanvas """
return mem_address(super(Stack, self).__repr__())
def _gen_keys(self, length):
""" Return a list of itemlables (item0, item1 etc...) using
self.itemlabel and a length"""
logger.debug('Items not found on %s: generating item list' %
self.full_name)
return [self.itemlabel + str(i) for i in range(length)]
# --------------------
# Dictionary Interface
def __getitem__(self, keyslice):
""" If single name, used dict interface. If slice or integer, uses
list interface. All results parameterized to key, data pairs, passed
directly into a new Stack.
"""
# Slice as list of strings or int [0, 'foo', 2, 'bar']
if hasattr(keyslice, '__iter__'):
tuples_out = []
for item in keyslice:
if isinstance(item, str):
item = self._data.keys().index(item)
tuples_out.append(self._data.items()[item])
else:
if isinstance(keyslice, int) or isinstance(keyslice, slice):
tuples_out = self._data.items()[keyslice]
else:
tuples_out = [(keyslice, self._data[keyslice])
] #keyslice is name
# If single item, return TimeSpectra, else, return new Stack
# Canonical slicing implementaiton; don't change unless good reason
# Because len() wonky with nested tuples (eg (x,y) and [(x1,y1),(x2,y2)]
# are both length two, this will work:
if sum(1 for x in tuples_out) == 2:
return tuples_out[1] #Return timespectra
else:
return self.__class__(tuples_out)
def __delitem__(self, keyslice):
""" Delete a single name, or a keyslice from names/canvas """
if isinstance(keyslice, str):
idx = self.names.index(keyslice)
self.pop(idx)
else:
raise NotImplementedError("Deletion only supports single entry")
def __setitem__(self, name, canvas):
""" """
if name in self.names:
idx = self.names.index(name)
self.pop(idx)
self.insert(idx, name, canvas)
else:
self.names.append(name)
def __getattr__(self, attr):
""" If attribute not found, try attribute lookup in dictionary. If
that is not found, try finding attribute on self._data.
For example, self.keys() will first look for self['keys']. Since
this isn't found, it calls self._data.keys(). But if I do
self.Item1, then it returns self['Item1']. The very rare conflict
case that a user has named the items a method that may already exist
in the dictionary (eg items=['a','b','keys'] is addressed.
"""
if attr in self._data.keys():
if hasattr(self._data, attr):
raise AttributeError('"%s attribute" found in both the items\
and as a method of the underlying dictionary object.' % (attr))
else:
return self[attr]
return getattr(self._data, attr)
# Attributes deferred to self.data /dictionary
def __len__(self):
return self._data.__len__()
def __iter__(self):
return self._data.__iter__()
def __reversed__(self):
return self._data.__reversed__()
def __contains__(self):
return self._data.__contains__()
def as_3d(self):
""" Return 3d structure of data. Default is panel."""
raise Panel(data=self._data)
### Data types without labels
#Is this realy necessary? See pyparty.ParticleManger for possibly more consistent implementation
def get_all(self, attr, astype=tuple):
"""Generator/tuple etc.. of (item, attribute) pairs. """
return put._parse_generator(((item[0], getattr(item[1], attr))
for item in self.items()), astype)
def _get_unique(self, attr):
""" Inspects Stack itemwise for an attribute for unique values.
If non-unique value for the attributes are found, returns
"mixed".
"""
unique = set(self.get_all(attr, astype=dict).values())
if len(unique) > 1:
return 'mixed'
else:
return tuple(unique)[0] #set doesn't support indexing
def set_all(self, attr, val, inplace=False):
""" Set attributes itemwise.
If not inplace, returns new instance of self"""
if inplace:
for (key, item) in self.items():
try:
setattr(item, attr, val)
except Exception as E:
raise Exception(
'Could not set %s in "%s". Received the following \
exception:\n "%s"' % (attr, key, E))
else:
out = deepcopy(self._data) #DEEPCOPY
for item in out:
setattr(out[item], attr, val)
return self.__class__(out)
def apply(self, func, *args, **kwargs):
""" Applies a user-passed function, or calls an instance method itemwise.
Parameters:
-----------
func: str or function
If string, must correspond to a method on the object stored
itemwise in the stack. If a function, appliked itemwise to
objects stored.
inplace: False
Special kwarg. If true, self._data modified inplace,
otherwise new specstack is returned.
*args, **kwargs:
func arguments.
Returns:
--------
If not inplace, returns SpecStack after itemwise application.
"""
inplace = kwargs.pop('inplace', False)
if isinstance(func, basestring):
if inplace:
for item in self:
self[item] = getattr(self[item], func)(*args, **kwargs)
else:
return self.__class__(OrderedDict([(k, getattr(v, func)(*args, \
**kwargs)) for k,v in self.items()]))
# function, numpyfunction etc...
else:
if inplace:
for item in self:
self[item] = self[item].apply(func)(*args, **kwargs)
else:
return self.__class__(OrderedDict([(k, v.apply(func, *args, \
**kwargs)) for k,v in self.items()]))
@property
def full_name(self):
""" Timespectra:name or Timespectra:unnamed. Useful for scripts mostly """
outname = getattr(self, 'name', 'unnamed')
return '%s:%s' % (self.__class__.__name__, self.name)
class OrderedSet(MutableSet):
__slots__ = ("_map", )
def __init__(self, iterable=None):
# pylint: disable=super-init-not-called
# Always use OrderedDict as plain dict does not support
# __reversed__ and key indexing
self._map = OrderedDict()
# or is overridden in mutable set; calls add on each element
if iterable is not None:
self |= iterable
def add(self, value):
if value not in self._map:
self._map[value] = None
def discard(self, value):
if value in self._map:
self._map.pop(value)
def __iter__(self):
return self._map.__iter__()
def __reversed__(self):
return self._map.__reversed__()
def peek(self, last=True):
if not self._map: # i.e., is self._map empty?
raise KeyError('set is empty')
if last:
return next(reversed(self))
else:
return next(iter(self))
def __len__(self):
return len(self._map)
def __contains__(self, key):
return key in self._map
def update(self, iterable):
for item in iterable:
self.add(item)
def pop(self, last=True): # pylint: disable=arguments-differ
key = self.peek(last)
self.discard(key)
return key
def __repr__(self):
if not self._map: # i.e., is self._map empty?
return '%s()' % (self.__class__.__name__, )
return '%s(%r)' % (self.__class__.__name__, list(self))
def __eq__(self, other):
if isinstance(other, OrderedSet):
return len(self) == len(other) and self._map == other._map
return set(self) == set(other)
def __ne__(self, other):
""" Comparison method for comparing ordered sets
:param other: instance of OrderedSet
:rtype: None
"""
return not self.__eq__(other)
def __iter__(self) : return OrderedDict.__reversed__(self) def __reversed__(self) : return OrderedDict.__iter__(self)
def __reversed__(self):
return OrderedDict.__reversed__(self._dict)
class FormatFile:
"""
Represent SYN format file. Each object contains dictionary
{RegexObject: [RegexParam, RegexParam, ...]} and for the best
readability that dictionary elements can be accessed using
wrapper methods described below
"""
def __init__(self, filename):
"""Initializes FormatFile object"""
self._re_param_map = OrderedDict()
self.name = filename
try:
with open(filename, encoding='utf-8') as format_file:
for line in format_file:
fmt_line_pattern = r'^([^\t]+)\t+(\w[\w, \t:]+)\n?$'
match = re.match(fmt_line_pattern, line)
if not match:
raise FormatFileError('syntax error')
try:
norm_re_obj = _normalize_regex(match.group(1))
param_list = _get_param_list(match.group(2))
except FormatFileError as e:
raise e
else:
self._re_param_map[norm_re_obj] = param_list
except IOError as io_error:
io_error.strerror = 'cannot open file for reading'
raise io_error
except FormatFileError as synt_error:
raise synt_error
def __iter__(self):
"""Wrapper method to access format file elements"""
return self._re_param_map.__iter__()
def __next__(self):
"""Wrapper method to access format file elements"""
return self._re_param_map.__next__()
def __reversed__(self):
"""Wrapper method to access format file elements"""
return self._re_param_map.__reversed__()
def __getitem__(self, key):
"""Wrapper method to access format file elements"""
return self._re_param_map[key]
def __setitem__(self, key, value):
"""Wrapper method to access format file elements"""
self._re_param_map[key] = value
def __delitem__(self, key):
"""Wrapper method to access format file elements"""
del self._re_param_map[key]
def __missing__(self, nonexistent_key):
"""Wrapper method to access format file elements"""
nonexistent_key = '\'{0}\''.format(nonexistent_key)
raise Exception('regex ' + nonexistent_key + 'doesn\'t exist')
