Example #1
1
 def __getitem__(self, key):
     if OrderedDict.__contains__(self, key):
         return OrderedDict.__getitem__(self, key)
     elif (not OrderedDict.__contains__(self, key)) and (self.parent is not None):
         dct = find_next_dict(self)
         if dct is None:
             raise KeyError("%r not in dictionary" % key)
         else:
             return dct[key]
     else:
         raise KeyError("%r not in dictionary" % key)
Example #2
1
class Pipeline:
    def __init__(self, name, **args):
        self.name = name
        self.dict = OrderedDict(args)

    def __getitem__(self, key):
        return self.dict.__getitem__(key)

    def __setitem__(self, key, val):
        return self.dict.__setitem__(key, val)

    def has_key(self, key):
        return self.dict.__contains__(key)

    def __repr__(self):
        return "Pipeline(%s,%r)" % (self.name, ", ".join("%s=%s" % (key, val) for (key, val) in self.dict.items()))

    def flush(self):
        self.dict.clear()

    def stall(self, needed):
        stall = 0
        for x in needed:
            stall = max(stall, self.dict.get(x, 0))
        return stall

    def conflicts(self, needed):
        conflict = OrderedDict()
        for x in needed:
            if x in self.dict:
                conflict[x] = self.dict[x]
        return conflict

    def retire(self, cycles=1):
        dict_retire(self.dict, cycles)
Example #3
1
class Metadata(MutableMapping):
    def __init__(self, seq=None):
        self.dct = OrderedDict(seq) if seq else OrderedDict()

    def __contains__(self, key):
        return self.dct.__contains__(key)

    def __getitem__(self, key):
        return self.dct.__getitem__(key)

    def __setitem__(self, key, value):
        self.dct.__setitem__(key, value)

    def __delitem__(self, key):
        return self.dct.__delitem__(key)

    def __iter__(self):
        return self.dct.__iter__()

    def __len__(self):
        return self.dct.__len__()

    def __repr__(self):
        return repr(self.dct)

    def __str__(self):
        return str(self.dct)
class InfoRow:
    def __init__(self):
        self.dict = OrderedDict()

    def __getitem__(self, item):
        if isinstance(item, int):
            return self.dict.items()[item][1]
        return self.dict.__getitem__(item.lower())

    def __contains__(self, item):
        return self.dict.__contains__(item.lower())

    def __setitem__(self, key, value):
        if isinstance(key, int):
            self.dict.items()[key][1] = value
        self.dict.__setitem__(key.lower(), value)

    def get(self, k, d=None):
        if isinstance(k, int):
            try:
                return self.dict.items()[k][1]
            except:
                return d
        return self.dict.get(k.lower(), d)

    def __str__(self, rowSeparator="\n", columnSeparator="\t"):
        return getStr(dict2Str(self.dict))

    def __unicode__(self):
        return str(self)

    def __repr__(self):
        return str(self)
Example #5
1
class MetafeatureFunctions(object):
    def __init__(self):
        self.functions = OrderedDict()
        self.dependencies = OrderedDict()
        self.values = OrderedDict()

    def clear(self):
        self.values = OrderedDict()

    def __iter__(self):
        return self.functions.__iter__()

    def __getitem__(self, item):
        return self.functions.__getitem__(item)

    def __setitem__(self, key, value):
        return self.functions.__setitem__(key, value)

    def __delitem__(self, key):
        return self.functions.__delitem__(key)

    def __contains__(self, item):
        return self.functions.__contains__(item)

    def get_value(self, key):
        return self.values[key].value

    def set_value(self, key, item):
        self.values[key] = item

    def is_calculated(self, key):
        """Return if a helper function has already been executed.

        Necessary as get_value() can return None if the helper function hasn't
        been executed or if it returned None."""
        return key in self.values

    def get_dependency(self, name):
        """Return the dependency of metafeature "name".
        """
        return self.dependencies.get(name)

    def define(self, name, dependency=None):
        """Decorator for adding metafeature functions to a "dictionary" of
        metafeatures. This behaves like a function decorating a function,
        not a class decorating a function"""

        def wrapper(metafeature_class):
            instance = metafeature_class()
            self.__setitem__(name, instance)
            self.dependencies[name] = dependency
            return instance

        return wrapper
Example #6
1
def getEsgfQuery(request_args):

    where = os.path.dirname(__file__)
    facetfile = os.path.join(where, "esgf-mars-facet-mapping")
    mappingsfile = os.path.join(where, "esgf-mars-default-mapping")
    attribsfile = os.path.join(where, "esgf-mars-attrib-mapping")
    mappingsdict = OrderedDict()
    attribsdict = OrderedDict()
    facetsdict = OrderedDict()
    fp = open(mappingsfile, "r")
    mappingsdict = json.load(fp, object_pairs_hook=OrderedDict)
    fp.close()
    fp = open(attribsfile, "r")
    attribsdict = json.load(fp, object_pairs_hook=OrderedDict)
    fp.close()
    fp = open(facetfile, "r")
    facetsdict = json.load(fp, object_pairs_hook=OrderedDict)
    fp.close()
    variablename = ""
    freqval = ""
    try:
        for facet, backend in facetsdict.iteritems():
            val = request_args.get(facet)
            if val != None:
                for backendopt in backend:
                    if attribsdict.__contains__(backendopt):
                        if facet == "variable":
                            variablename = val
                        if facet == "frequency":
                            freqval = val

                            # we have a match for the facet..
                        if attribsdict[backendopt].__contains__(val):
                            # we even have a substitution value for the specified facet value
                            # print 'backend value for user supplied facet value %s is %s'%(val,attribsdict[backendopt][val])
                            sub = attribsdict[backendopt][val]
                            # print 'before change: mappingsdict val=%s'%(mappingsdict[backendopt])
                            mappingsdict[backendopt] = sub
                            # print 'after change: mappingsdict val=%s'%(mappingsdict[backendopt])
                            # print "%s:%s"%(facet,val)
        mappingsdict["date"] = mappingsdict["datestr"]
        mappingsdict["date"] += str(mappingsdict["freq"])
        mappingsdict.pop("datestr")
        mappingsdict.pop("freq")
    except:
        raise
    fn = "%s_Eur05_SMHI-HIRLAM_RegRean_v0d0_SMHI-MESAN_v1_%s.grb" % (variablename, freqval)
    return (fn, mappingsdict)
class ChainSet(object):
    """
    Base class for various methods to rename chains

    Contains _chains, which maps from the renamed chain to a tuple with the
    original (object,state,chain). All dict-like accessors work on ChainSets,
    e.g.
        chain_set["A"] -> ("obj",1,"A")

    """

    def __init__(self):
        # Use an OrderedDict in Python >= 1.7 for better printing
        if _orderedDict:
            self._chains = OrderedDict()
        else:
            self._chains = dict()

    def map_chain(self, obj, state, origChain):
        """
        map_chain(string obj,int state, string chain]]) -> string

        Maps a chain letter to a unique chainID. Results are unique within each
        instance, and can be used as keys on this chain set.
        """
        raise NotImplementedError("Base class")

    # delegate most methods to _chains
    def __getattr__(self, at):
        if at in "pop popitem update setdefault".split():
            raise AttributeError("type object '%s' has no attribute '%s'" % (type(self), at))
        return getattr(self._chains, at)

    def __cmp__(self, other):
        return self._chains.__cmp__(other)

    def __eq__(self, other):
        return self._chains.__eq__(other)

    def __ge__(self, other):
        return self._chains.__ge__(other)

    def __gt__(self, other):
        return self._chains.__gt__(other)

    def __le__(self, other):
        return self._chains.__le__(other)

    def __lt__(self, other):
        return self._chains.__lt__(other)

    def __ne__(self, other):
        return self._chains.__ne__(other)

    def __len__(self):
        return self._chains.__len__()

    def __contains__(self, key):
        return self._chains.__contains__(key)

    def __getitem__(self, key):
        return self._chains.__getitem__(key)

    def __iter__(self):
        return self._chains.__iter__()

    def __str__(self):
        return str(self._chains)

    @staticmethod
    def _int_to_chain(i, base=_default_base):
        """
        _int_to_chain(int,int) -> str

        Converts a positive integer to a chain ID. Chain IDs include uppercase
        characters, numbers, and optionally lowercase letters.

        i = a positive integer to convert
        base = the alphabet size to include. Typically 36 or 62.
        """
        if i < 0:
            raise ValueError("positive integers only")
        if base < 0 or 62 < base:
            raise ValueError("Invalid base")

        quot = int(i) / base
        rem = i % base
        if rem < 26:
            letter = chr(ord("A") + rem)
        elif rem < 36:
            letter = str(rem - 26)
        else:
            letter = chr(ord("a") + rem - 36)
        if quot == 0:
            return letter
        else:
            return ChainSet._int_to_chain(quot - 1, base) + letter
Example #8
1
class DotMap(OrderedDict):
    def __init__(self, *args, **kwargs):
        self._map = OrderedDict()
        if args:
            d = args[0]
            if type(d) is dict:
                for k, v in self.__call_items(d):
                    if type(v) is dict:
                        v = DotMap(v)
                    self._map[k] = v
        if kwargs:
            for k, v in self.__call_items(kwargs):
                self._map[k] = v

    def __call_items(self, obj):
        if hasattr(obj, "iteritems") and ismethod(getattr(obj, "iteritems")):
            return obj.iteritems()
        else:
            return obj.items()

    def items(self):
        return self.iteritems()

    def iteritems(self):
        return self.__call_items(self._map)

    def __iter__(self):
        return self._map.__iter__()

    def next(self):
        return self._map.next()

    def __setitem__(self, k, v):
        self._map[k] = v

    def __getitem__(self, k):
        if k not in self._map:
            # automatically extend to new DotMap
            self[k] = DotMap()
        return self._map[k]

    def __setattr__(self, k, v):
        if k == "_map":
            super(DotMap, self).__setattr__(k, v)
        else:
            self[k] = v

    def __getattr__(self, k):
        if k == "_map":
            super(DotMap, self).__getattr__(k)
        else:
            return self[k]

    def __delattr__(self, key):
        return self._map.__delitem__(key)

    def __contains__(self, k):
        return self._map.__contains__(k)

    def __str__(self):
        items = []
        for k, v in self.__call_items(self._map):
            items.append("{0}={1}".format(k, repr(v)))
        out = "DotMap({0})".format(", ".join(items))
        return out

    def __repr__(self):
        return str(self)

    def toDict(self):
        d = {}
        for k, v in self.items():
            if type(v) is DotMap:
                v = v.toDict()
            d[k] = v
        return d

    def pprint(self):
        pprint(self.toDict())

        # proper dict subclassing

    def values(self):
        return self._map.values()

    @classmethod
    def parseOther(self, other):
        if type(other) is DotMap:
            return other._map
        else:
            return other

    def __cmp__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__cmp__(other)

    def __eq__(self, other):
        other = DotMap.parseOther(other)
        if not isinstance(other, dict):
            return False
        return self._map.__eq__(other)

    def __ge__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__ge__(other)

    def __gt__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__gt__(other)

    def __le__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__le__(other)

    def __lt__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__lt__(other)

    def __ne__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__ne__(other)

    def __delitem__(self, key):
        return self._map.__delitem__(key)

    def __len__(self):
        return self._map.__len__()

    def clear(self):
        self._map.clear()

    def copy(self):
        return self

    def get(self, key, default=None):
        return self._map.get(key, default)

    def has_key(self, key):
        return key in self._map

    def iterkeys(self):
        return self._map.iterkeys()

    def itervalues(self):
        return self._map.itervalues()

    def keys(self):
        return self._map.keys()

    def pop(self, key, default=None):
        return self._map.pop(key, default)

    def popitem(self):
        return self._map.popitem()

    def setdefault(self, key, default=None):
        self._map.setdefault(key, default)

    def update(self, *args, **kwargs):
        if len(args) != 0:
            self._map.update(*args)
        self._map.update(kwargs)

    def viewitems(self):
        return self._map.viewitems()

    def viewkeys(self):
        return self._map.viewkeys()

    def viewvalues(self):
        return self._map.viewvalues()

    @classmethod
    def fromkeys(cls, seq, value=None):
        d = DotMap()
        d._map = OrderedDict.fromkeys(seq, value)
        return d
Example #9
1
class SortedDotDict(object):
    def __init__(self, *args, **kwargs):
        super(SortedDotDict, self).__init__(*args, **kwargs)
        self._dict = SortedDict()

    def __contains__(self, *args, **kwargs):
        return self._dict.__contains__(*args, **kwargs)

    def __eq__(self, *args, **kwargs):
        return self._dict.__eq__(*args, **kwargs)

    def __format__(self, *args, **kwargs):
        return self._dict.__format__(*args, **kwargs)

    def __ge__(self, *args, **kwargs):
        return self._dict.__ge__(*args, **kwargs)

    def __getattr__(self, key):
        try:
            return self._dict[key]
        except:
            raise AttributeError(key)

    def __iter__(self):
        vals = list(self.values())
        for k in vals:
            yield k

    def __getitem__(self, key):
        return self._dict[key]

    def __setitem__(self, key, value):
        self._dict[key] = value

    def __delitem__(self, key):
        del self._dict[key]

    def keys(self):
        return list(self._dict.keys())

    def values(self):
        vals = list(self._dict.values())
        vals = [v for v in vals if isinstance(v, (ConfigurationGroup, Value))]
        vals.sort()
        return vals

    def items(self):
        return list(self._dict.items())

    def iterkeys(self):
        return iter(self._dict.keys())

    def itervalues(self):
        return iter(self._dict.values())

    def iteritems(self):
        return iter(self._dict.items())

    def get(self, *args, **kwargs):
        return self._dict.get(*args, **kwargs)

    def clear(self):
        return self._dict.clear()

    def copy(self):
        s = SortedDotDict()
        s._dict = self._dict.copy()
        return s

    def fromkeys(self):
        return self._dict.fromkeys()

    def has_key(self, key):
        return key in self._dict

    def pop(self, *args, **kwargs):
        return self._dict.pop(*args, **kwargs)

    def popitem(self, *args, **kwargs):
        return self._dict.popitem(*args, **kwargs)

    def setdefault(self, key, default):
        return self._dict.setdefault(key, default)

    def update(self, d):
        return self._dict.update(d)

    def viewitems(self, *args, **kwargs):
        return self._dict.viewitems(*args, **kwargs)

    def viewvalues(self, *args, **kwargs):
        return self._dict.viewvalues(*args, **kwargs)
Example #10
1
class DotMap(MutableMapping, OrderedDict):
    def __init__(self, *args, **kwargs):
        self._map = OrderedDict()
        self._dynamic = True
        if kwargs:
            if "_dynamic" in kwargs:
                self._dynamic = kwargs["_dynamic"]
        if args:
            d = args[0]
            if isinstance(d, dict):
                for k, v in self.__call_items(d):
                    if type(v) is dict:
                        v = DotMap(v, _dynamic=self._dynamic)
                    if type(v) is list:
                        l = []
                        for i in v:
                            n = i
                            if type(i) is dict:
                                n = DotMap(i, _dynamic=self._dynamic)
                            l.append(n)
                        v = l
                    self._map[k] = v
        if kwargs:
            for k, v in self.__call_items(kwargs):
                if k is not "_dynamic":
                    self._map[k] = v

    def __call_items(self, obj):
        if hasattr(obj, "iteritems") and ismethod(getattr(obj, "iteritems")):
            return obj.iteritems()
        else:
            return obj.items()

    def items(self):
        return self.iteritems()

    def iteritems(self):
        return self.__call_items(self._map)

    def __iter__(self):
        return self._map.__iter__()

    def next(self):
        return self._map.next()

    def __setitem__(self, k, v):
        self._map[k] = v

    def __getitem__(self, k):
        if k not in self._map and self._dynamic and k != "_ipython_canary_method_should_not_exist_":
            # automatically extend to new DotMap
            self[k] = DotMap()
        return self._map[k]

    def __setattr__(self, k, v):
        if k in {"_map", "_dynamic", "_ipython_canary_method_should_not_exist_"}:
            super(DotMap, self).__setattr__(k, v)
        else:
            self[k] = v

    def __getattr__(self, k):
        if k == {"_map", "_dynamic", "_ipython_canary_method_should_not_exist_"}:
            super(DotMap, self).__getattr__(k)
        else:
            return self[k]

    def __delattr__(self, key):
        return self._map.__delitem__(key)

    def __contains__(self, k):
        return self._map.__contains__(k)

    def __str__(self):
        items = []
        for k, v in self.__call_items(self._map):
            # bizarre recursive assignment situation (why someone would do this is beyond me)
            if id(v) == id(self):
                items.append("{0}=DotMap(...)".format(k))
            else:
                items.append("{0}={1}".format(k, repr(v)))
        out = "DotMap({0})".format(", ".join(items))
        return out

    def __repr__(self):
        return str(self)

    def toDict(self):
        d = {}
        for k, v in self.items():
            if type(v) is DotMap:
                # bizarre recursive assignment support
                if id(v) == id(self):
                    v = d
                else:
                    v = v.toDict()
            elif type(v) is list:
                l = []
                for i in v:
                    n = i
                    if type(i) is DotMap:
                        n = i.toDict()
                    l.append(n)
                v = l
            d[k] = v
        return d

    def pprint(self):
        pprint(self.toDict())

    def empty(self):
        return not any(self)

        # proper dict subclassing

    def values(self):
        return self._map.values()

        # ipython support

    def __dir__(self):
        return self.keys()

    @classmethod
    def parseOther(self, other):
        if type(other) is DotMap:
            return other._map
        else:
            return other

    def __cmp__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__cmp__(other)

    def __eq__(self, other):
        other = DotMap.parseOther(other)
        if not isinstance(other, dict):
            return False
        return self._map.__eq__(other)

    def __ge__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__ge__(other)

    def __gt__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__gt__(other)

    def __le__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__le__(other)

    def __lt__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__lt__(other)

    def __ne__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__ne__(other)

    def __delitem__(self, key):
        return self._map.__delitem__(key)

    def __len__(self):
        return self._map.__len__()

    def clear(self):
        self._map.clear()

    def copy(self):
        return DotMap(self.toDict())

    def __copy__(self):
        return self.copy()

    def __deepcopy__(self, memo=None):
        return self.copy()

    def get(self, key, default=None):
        return self._map.get(key, default)

    def has_key(self, key):
        return key in self._map

    def iterkeys(self):
        return self._map.iterkeys()

    def itervalues(self):
        return self._map.itervalues()

    def keys(self):
        return self._map.keys()

    def pop(self, key, default=None):
        return self._map.pop(key, default)

    def popitem(self):
        return self._map.popitem()

    def setdefault(self, key, default=None):
        self._map.setdefault(key, default)

    def update(self, *args, **kwargs):
        if len(args) != 0:
            self._map.update(*args)
        self._map.update(kwargs)

    def viewitems(self):
        return self._map.viewitems()

    def viewkeys(self):
        return self._map.viewkeys()

    def viewvalues(self):
        return self._map.viewvalues()

    @classmethod
    def fromkeys(cls, seq, value=None):
        d = DotMap()
        d._map = OrderedDict.fromkeys(seq, value)
        return d

    def __getstate__(self):
        return self.__dict__

    def __setstate__(self, d):
        self.__dict__.update(d)
Example #11
0
class ArgumentsGroup(BaseType):
    def __init__(self, *args, **kwargs):
        arguments = to_arguments(*args, **kwargs)
        self.arguments = OrderedDict([(arg.name, arg) for arg in arguments])

    def internal_type(self, schema):
        return OrderedDict([(arg.name, schema.T(arg)) for arg in self.arguments.values()])

    def __len__(self):
        return len(self.arguments)

    def __iter__(self):
        return iter(self.arguments)

    def __contains__(self, *args):
        return self.arguments.__contains__(*args)

    def __getitem__(self, *args):
        return self.arguments.__getitem__(*args)
Example #12
0
    def __contains__(self, key):
        """
        Checks if given key exists in by first trying to find an exact match.
        If no exact match is found then this method iterates trhough keys
        and tries to check if given key is either:

            1. A subclass of one of the keys
            2. An instance of one of the keys

        The first check has the time complexity of O(1) whereas the second
        check has O(n).

        Example::



            class A(object):
                pass


            class B(object):
                pass


            class A2(A):
                pass


            class_map = ClassMap({A: 1, B: 2})
            assert B in class_map
            assert A in class_map
            assert A2 in class_map
            assert B() in class_map
            assert A() in class_map
            assert A2() in class_map
        """
        if OrderedDict.__contains__(self, key):
            return True
        test_func = issubclass if isclass(key) else isinstance
        return any(test_func(key, class_) for class_ in self)
Example #13
0
class redict(object):
    """
        @summary: Class to store regexes as dictionary keys
    """

    def __init__(self):
        self._dict = OrderedDict()

    def __setitem__(self, key, value):
        self._dict[key] = [re.compile(key[0], key[1]), value]

    def __getitem__(self, key):
        return self._dict[key]

    def __delitem__(self, key):
        self._dict.pop(key)

    def __contains__(self, k):
        return k in self._dict.__contains__(k)

    has_key = __contains__

    def __iter__(self):
        for k in self._dict.iterkeys():
            yield k

    iterkeys = __iter__

    def iteritems(self):
        for (k, v) in self._dict.iteritems():
            yield (k, v)

    def itervalues(self):
        for v in self._dict.itervalues():
            yield v

    # Extended functions
    def iter_regex(self):
        for (v1, v2) in self._dict.itervalues():
            yield (v1, v2)
Example #14
0
def OPTIMO(wlf, n):
    pages = []
    # Read pages from file.
    f = open(wlf)
    pages = f.read().splitlines()
    f.close()
    cache_list = OrderedDict()
    page_fault = 0
    count = 0
    # Insertion of pages to memory and calculation of page faults.
    for page in pages:
        tam_cache = len(cache_list)
        lista = []
        count += 1
        if not cache_list.__contains__(page):
            if tam_cache < n:
                cache_list[page] = ""
                page_fault += 1
            else:
                lista = [pages[x] for x in range(count + 1, len(pages))]
                list_inters = set(lista).intersection(cache_list)
                if len(list_inters) == len(cache_list):
                    cache_list.popitem()
                    cache_list[page] = ""
                    page_fault += 1
                else:
                    lista_comp = set(list_inters).symmetric_difference(cache_list)
                    elem = lista_comp.pop()
                    cache_list.__delitem__(elem)
                    cache_list[page] = ""
                    page_fault += 1
    return page_fault
    print "Resultados: "
    print "Miss rate: ", "              " + str(round((float(page_fault * 100) / (3721736)), 3)) + "%"
    print "Miss rate (warm cache): ", " " + str(round((float(page_fault * 100) / (3721736 - n)), 3)) + "%"
    with open("log.csv", "a") as output:
        output.write("OPTIMO," + str(page_fault) + "," + str(n) + "\n")
    output.close()
Example #15
0
 def __contains__(self, key):
     """Overide __contains__"""
     return OrderedDict.__contains__(self, key.lower())
from collections import OrderedDict

d = OrderedDict()
for tc in range(int(input())):
    key = input()
    if d.__contains__(key):
        d[key] += 1
    else:
        d[key] = 1
print(len(d))
print(" ".join(map(lambda x: str(d[x]), d)))
Example #17
0
 def __contains__(self, key):
     if OrderedDict.__contains__(self, key):
         return True
     return self.parent and key in self.parent
Example #18
0
class Stack(object):
    """ Base class to store pandas objects, with special operations to
    return as 3d data (eg panel) and to apply functions itemwise.  Items are
    stored in an ordered dict."""

    itemlabel = "Item"

    _magic = ["__len__", "__iter__", "__reversed__", "__contains__"]

    def __init__(self, data, keys=None, name="", sort_items=False):
        self.name = name

        # Dictionary input
        if isinstance(data, dict):
            logger.debug('Initializing "%s" from dictionary.' % self.full_name)
            if sort_items:
                logger.debug("Sorting keys")
                self._data = OrderedDict(sorted(data.keys(), key=lambda t: t[0]))

            else:
                self._data = OrderedDict(data)

        else:

            if not isinstance(data, Iterable):
                logger.info("%s constructed from non-iterable... converting " "data to an iterable" % self.full_name)
                data = [data]

            if keys:
                if not isinstance(keys, Iterable):
                    logger.info(
                        "%s constructed from non-iterable... converting " "keys to an iterable" % self.full_name
                    )
                    keys = [keys]

                if len(keys) != len(data):
                    raise ValueError("Length mistmatch: keys and data (%s,%s)" % (len(keys), len(data)))

            # If keys not passed, generate them
            else:
                # Zipped data ((key, df), (key, df))
                try:
                    keys, data = zip(*data)
                except Exception:
                    keys = self._gen_keys(len(data))
                    if len(keys) > 1:
                        logger.warn("Generating keys %s-%s" % (keys[0], keys[-1]))
                    else:
                        logger.warn("Generating key %s" % keys[0])

            self._data = OrderedDict([(key, data[i]) for (i, key) in enumerate(keys)])

    @property
    def _address(self):
        """ Property to make easily accesible by multicanvas """
        return mem_address(super(Stack, self).__repr__())

    def _gen_keys(self, length):
        """ Return a list of itemlables (item0, item1 etc...) using
            self.itemlabel and a length"""

        logger.debug("Items not found on %s: generating item list" % self.full_name)
        return [self.itemlabel + str(i) for i in range(length)]

    # --------------------
    # Dictionary Interface
    def __getitem__(self, keyslice):
        """ If single name, used dict interface.  If slice or integer, uses 
        list interface.  All results parameterized to key, data pairs, passed
        directly into a new Stack.
        """
        # Slice as list of strings or int [0, 'foo', 2, 'bar']
        if hasattr(keyslice, "__iter__"):

            tuples_out = []
            for item in keyslice:
                if isinstance(item, str):
                    item = self._data.keys().index(item)
                tuples_out.append(self._data.items()[item])

        else:
            if isinstance(keyslice, int) or isinstance(keyslice, slice):
                tuples_out = self._data.items()[keyslice]
            else:
                tuples_out = [(keyslice, self._data[keyslice])]  # keyslice is name

        # If single item, return TimeSpectra, else, return new Stack
        # Canonical slicing implementaiton; don't change unless good reason
        # Because len() wonky with nested tuples (eg (x,y) and [(x1,y1),(x2,y2)]
        # are both length two, this will work:

        if sum(1 for x in tuples_out) == 2:
            return tuples_out[1]  # Return timespectra
        else:
            return self.__class__(tuples_out)

    def __delitem__(self, keyslice):
        """ Delete a single name, or a keyslice from names/canvas """

        if isinstance(keyslice, str):
            idx = self.names.index(keyslice)
            self.pop(idx)
        else:
            raise NotImplementedError("Deletion only supports single entry")

    def __setitem__(self, name, canvas):
        """ """
        if name in self.names:
            idx = self.names.index(name)
            self.pop(idx)
            self.insert(idx, name, canvas)

        else:
            self.names.append(name)

    def __getattr__(self, attr):
        """ If attribute not found, try attribute lookup in dictionary.  If
        that is not found, try finding attribute on self._data.
        
        For example, self.keys() will first look for self['keys'].  Since
        this isn't found, it calls self._data.keys().  But if I do 
        self.Item1, then it returns self['Item1'].  The very rare conflict
        case that a user has named the items a method that may already exist
        in the dictionary (eg items=['a','b','keys'] is addressed.
        """

        if attr in self._data.keys():
            if hasattr(self._data, attr):
                raise AttributeError(
                    '"%s attribute" found in both the items\
                and as a method of the underlying dictionary object.'
                    % (attr)
                )
            else:
                return self[attr]
        return getattr(self._data, attr)

    # Attributes deferred to self.data /dictionary
    def __len__(self):
        return self._data.__len__()

    def __iter__(self):
        return self._data.__iter__()

    def __reversed__(self):
        return self._data.__reversed__()

    def __contains__(self):
        return self._data.__contains__()

    def as_3d(self):
        """ Return 3d structure of data.  Default is panel."""
        raise Panel(data=self._data)

        ### Data types without labels

    # Is this realy necessary?  See pyparty.ParticleManger for possibly more consistent implementation
    def get_all(self, attr, astype=tuple):
        """Generator/tuple etc.. of (item, attribute) pairs. """

        return put._parse_generator(((item[0], getattr(item[1], attr)) for item in self.items()), astype)

    def _get_unique(self, attr):
        """ Inspects Stack itemwise for an attribute for unique values.
        If non-unique value for the attributes are found, returns
        "mixed". 
        """
        unique = set(self.get_all(attr, astype=dict).values())
        if len(unique) > 1:
            return "mixed"
        else:
            return tuple(unique)[0]  # set doesn't support indexing

    def set_all(self, attr, val, inplace=False):
        """ Set attributes itemwise.  
            If not inplace, returns new instance of self"""
        if inplace:
            for (key, item) in self.items():
                try:
                    setattr(item, attr, val)
                except Exception as E:
                    raise Exception(
                        'Could not set %s in "%s".  Received the following \
                     exception:\n "%s"'
                        % (attr, key, E)
                    )
        else:
            out = deepcopy(self._data)  # DEEPCOPY
            for item in out:
                setattr(out[item], attr, val)
            return self.__class__(out)

    def apply(self, func, *args, **kwargs):
        """ Applies a user-passed function, or calls an instance method itemwise.
        
        
        Parameters:
        -----------
        func: str or function
            If string, must correspond to a method on the object stored 
            itemwise in the stack.  If a function, appliked itemwise to
            objects stored.  
              
        inplace: False 
            Special kwarg.  If true, self._data modified inplace, 
            otherwise new specstack is returned.
                         
        *args, **kwargs: 
            func arguments.
          
        Returns:
        --------
        If not inplace, returns SpecStack after itemwise application.
            
        """

        inplace = kwargs.pop("inplace", False)

        if isinstance(func, basestring):
            if inplace:
                for item in self:
                    self[item] = getattr(self[item], func)(*args, **kwargs)

            else:
                return self.__class__(OrderedDict([(k, getattr(v, func)(*args, **kwargs)) for k, v in self.items()]))

        # function, numpyfunction etc...
        else:
            if inplace:
                for item in self:
                    self[item] = self[item].apply(func)(*args, **kwargs)

            else:
                return self.__class__(OrderedDict([(k, v.apply(func, *args, **kwargs)) for k, v in self.items()]))

    @property
    def full_name(self):
        """ Timespectra:name or Timespectra:unnamed.  Useful for scripts mostly """
        outname = getattr(self, "name", "unnamed")
        return "%s:%s" % (self.__class__.__name__, self.name)
Example #19
0
def create_index():

    ## stopwords from nltk.corpus
    stopwords = [
        "i",
        "me",
        "my",
        "myself",
        "we",
        "our",
        "ours",
        "ourselves",
        "you",
        "your",
        "yours",
        "yourself",
        "yourselves",
        "he",
        "him",
        "his",
        "himself",
        "she",
        "her",
        "hers",
        "herself",
        "it",
        "its",
        "itself",
        "they",
        "them",
        "their",
        "theirs",
        "themselves",
        "what",
        "which",
        "who",
        "whom",
        "this",
        "that",
        "these",
        "those",
        "am",
        "is",
        "are",
        "was",
        "were",
        "be",
        "been",
        "being",
        "have",
        "has",
        "had",
        "having",
        "do",
        "does",
        "did",
        "doing",
        "a",
        "an",
        "the",
        "and",
        "but",
        "if",
        "or",
        "because",
        "as",
        "until",
        "while",
        "of",
        "at",
        "by",
        "for",
        "with",
        "about",
        "against",
        "between",
        "into",
        "through",
        "during",
        "before",
        "after",
        "above",
        "below",
        "to",
        "from",
        "up",
        "down",
        "in",
        "out",
        "on",
        "off",
        "over",
        "under",
        "again",
        "further",
        "then",
        "once",
        "here",
        "there",
        "when",
        "where",
        "why",
        "how",
        "all",
        "any",
        "both",
        "each",
        "few",
        "more",
        "most",
        "other",
        "some",
        "such",
        "no",
        "nor",
        "not",
        "only",
        "own",
        "same",
        "so",
        "than",
        "too",
        "very",
        "s",
        "t",
        "can",
        "will",
        "just",
        "don",
        "should",
        "now",
    ]

    print "creating index.."
    start = time.clock()
    path1 = ".//Extracted_text_files"
    dirlist = os.listdir(path1)  ## list all the extracted text files
    index = OrderedDict()
    porter = nltk.PorterStemmer()
    for input_file in dirlist:  ## for each text file
        count = 0
        docid = input_file.strip()
        with open(path1 + "//" + input_file, "r") as input_line:  ## open the text file in read mode
            lines = input_line.read().strip()
            words = re.findall(r"\w+", lines, flags=re.UNICODE | re.LOCALE)  ## extract words from the text
            for word in words:  ## for each word
                count += 1  ## count is calculated to get the word position, hence count indicates the word's position
                word = word.lower().strip()  ## lower-case the word
                word = porter.stem(word)  ## stemming the words before inserting into the index
                if index.__contains__(word):  ## check if index contains the word
                    if index[word].__contains__(docid):  ## check if there is a doc containing this word
                        index[word][docid].append(count)  ## if yes, append the new position to previous positions
                    else:  ## else create an entry for this new doc containing this word
                        index[word][docid] = []  ## initialize the value for this docid and
                        index[word][docid].append(count)  ## insert the position into the index entry against this docid
                else:  ## if word not in index, create an index entry for this word
                    index[word] = OrderedDict()
                    index[word][docid] = []
                    index[word][docid].append(count)  ## insert the position against the current docid

    for word in stopwords:  ## removing stop-words from the index
        word = porter.stem(word)
        if word in index.keys():
            del index[word]

    ## pickle the index dictionary for querying and ranking
    pickle.dump(index, open("index", "wb"))
    elapsed = time.clock() - start
    print elapsed
Example #20
0
 def __contains__(self, key):
     l = _lower(key)
     return OrderedDict.__contains__(self, l)
Example #21
0
File: yuri.py Project: jparise/yuri
 def __contains__(self, name):
     name = name.lower()
     return OrderedDict.__contains__(self, name)
Example #22
0
        for line in f:
            if cache.__contains__(line):
                # HIT
                cache.__delitem__(line)
                cache[line] = ""
                hites = hites + 1
            else:
                # MISS
                misses = misses + 1
                cache[line] = ""
                if len(cache) > cache_tam:
                    cache.popitem(last=False)
    references = hites + misses
    print "Resultados: "
    print "Miss rate: ", "              " + str(round((float(misses) / (references)), 3)) + "% (" + str(
        misses
    ) + " misses out of " + str(references) + " references)"
    print "Miss rate (warm cache): ", " " + str(round((float(misses) / (references - cache_tam)), 3)) + "% (" + str(
        hites
    ) + " misses out of " + str(references - cache_tam) + " references)"
    print "Efficiency: " + str(round(float(hites) / (references), 3))
    print "Hits: " + str(hites)
    print "Misses: " + str(misses)
    with open("log.txt", "a") as myfile:
        myfile.write(str(cache_tam) + "," + str(misses) + ",LRU\n")

elif policy.upper() == "FIFO":
    print "Evaluando una caché FIFO con " + str(cache_tam) + " entradas..."
    with open(workload_file) as f:
        for line in f:
            if cache.__contains__(line):
                # HIT