Example #1
1
class Records(object):
    def __init__(self):
        self._data = OrderedDict()
        self._stamps = {}

    def __iter__(self):
        return self._data.iteritems()

    def __contains__(self, key):
        return key in self._data

    def keys(self):
        return self._data.keys()

    def update(self, data):
        self._data.update(data)
        self._stamps.update(dict.fromkeys(data.keys(), time()))

    def pluck(self, t):
        return OrderedDict(filter(lambda item: self._stamps[item[0]] >= t, self._data.items()))

    def clear(self):
        self._data.clear()
        self._stamps.clear()

    def get(self, key):
        return self._data[key]
Example #2
1
def packet_summary(pkts, pcap):
    packet = OrderedDict()
    count = 1
    try:
        for p in pkts:
            p_header = {
                "Buffer": {
                    "timestamp": datetime.datetime.fromtimestamp(p.time).strftime("%Y-%m-%d %H:%M:%S.%f"),
                    "packetnumber": count,
                    "pcapfile": pcap,
                    "packet_length": p.len,
                }
            }
            packet.update(p_header)
            if p.haslayer(Ether):
                p_ether = {"Ethernet": {"ether_src": p[Ether].src, "ether_dst": p[Ether].dst}}
                packet.update(p_ether)
            if p.haslayer(IP):
                p_ip = {"IP": {"ip_src": p[IP].src, "ip_dst": p[IP].dst, "ip_ttl": p[IP].ttl}}
                packet.update(p_ip)
            if p.haslayer(TCP):
                p_tcp = {"TCP": {"tcp_sport": p[TCP].sport, "tcp_dport": p[TCP].dport, "tcp_flags": p[TCP].flags}}
                packet.update(p_tcp)
            if p.haslayer(UDP):
                p_udp = {"UDP": {"udp_sport": p[UDP].sport, "udp_dport": p[UDP].dport, "udp_len": p[UDP].len}}
                packet.update(p_udp)
            count += 1
            yield packet
            packet.clear()
    except Exception, e:
        print e
        pass
Example #3
1
class AliasGenerator(object):
    def __init__(self, size=200):
        self.MAX_CACHE = size  # TODO: Make configurable
        self.cache = OrderedDict()

    def generate_alias(self, str_value):
        """
        Generate an alias for the given string.
        :param str_value: the given string
        :return: a 24 character alias string
        """
        alias = self.get_alias(str_value)
        if not alias:
            h = hashlib.md5()
            h.update(str_value.encode("utf-8"))
            alias = h.hexdigest()
            self.cache[str_value] = alias
            if len(self.cache) > self.MAX_CACHE:
                self.cache.popitem(last=False)  # FIFO
        else:
            alias = alias
        return alias

    def needs_alias(self, str_value):
        return len(str_value) > MAX_NAME_LEN or len(RE_CLEANSE.findall(str_value)) > 0

    def get_alias(self, str_value):
        return self.cache.get(str_value)

    def clear_aliases(self):
        self.cache.clear()
class DownloadCore:
    """"""

    def __init__(self):
        """"""
        self.active_downloads = {}
        self.queue_downloads = OrderedDict()
        self.complete_downloads = {}
        self.stopped_downloads = {}

    def reorder_queue(self, id_order_list):
        """
        TODO: only replace the order list in OrderedDict.
        """
        ordered_items_dict = OrderedDict()
        for id_item in id_order_list:
            try:
                ordered_items_dict[id_item] = self.queue_downloads[id_item]
            except KeyError:
                pass
        if len(self.queue_downloads) == len(ordered_items_dict):
            self.queue_downloads.clear()
            self.queue_downloads.update(ordered_items_dict)
        else:
            logger.warning("reorder_queue failed")
Example #5
1
class Pipeline:
    def __init__(self, name, **args):
        self.name = name
        self.dict = OrderedDict(args)

    def __getitem__(self, key):
        return self.dict.__getitem__(key)

    def __setitem__(self, key, val):
        return self.dict.__setitem__(key, val)

    def has_key(self, key):
        return self.dict.__contains__(key)

    def __repr__(self):
        return "Pipeline(%s,%r)" % (self.name, ", ".join("%s=%s" % (key, val) for (key, val) in self.dict.items()))

    def flush(self):
        self.dict.clear()

    def stall(self, needed):
        stall = 0
        for x in needed:
            stall = max(stall, self.dict.get(x, 0))
        return stall

    def conflicts(self, needed):
        conflict = OrderedDict()
        for x in needed:
            if x in self.dict:
                conflict[x] = self.dict[x]
        return conflict

    def retire(self, cycles=1):
        dict_retire(self.dict, cycles)
 def test_clear(self):
     pairs = [("c", 1), ("b", 2), ("a", 3), ("d", 4), ("e", 5), ("f", 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     self.assertEqual(len(od), len(pairs))
     od.clear()
     self.assertEqual(len(od), 0)
class memoize_mask(object):
    """
    Decorator. Caches wktToMask keyed to model_id and the WKT string
    """

    def __init__(self, func, maxsize=50):
        """
        Args:
            func: the function to wrap
            maxsize (int): Max size of cache (in MB)
        """

        self.hits = self.misses = 0
        self.func = func
        self.maxsize = maxsize
        self.cache = OrderedDict()

    def __call__(self, *args):

        nc, fname, wkt, varname = args
        # If we have no key, automatic cache miss
        if not hasattr(nc, "model_id"):
            log.debug("Cache MISS (attribute 'model_id' not found)")
            return self.func(*args)

        # Set key to model_id and wkt polygon
        key = (nc.model_id, wkt)
        log.debug("Checking cache for key %s", key)

        with cache_lock:
            try:
                result = self.cache[key]
                self.cache.move_to_end(key)  # record recent use of this key
                self.hits += 1
                log.debug("Cache HIT")
                return result
            except KeyError:
                pass

        log.debug("Cache MISS")
        result = self.func(*args)

        with cache_lock:
            self.cache[key] = result
            self.misses += 1
            while getsize(self.cache) > self.maxsize * 1024 * 1024:  # convert to MB
                self.cache.popitem(0)  # Purge least recently used cache entry

        return result

    def cache_clear(self):
        with cache_lock:
            self.cache.clear()
            self.hits = 0
            self.misses = 0
Example #8
0
class OrderedDict(object):
    def __init__(self, eq_func=None, hash_func=None):
        self.contents = PyOrderedDict()
        self.eq_func = eq_func or operator.eq
        self.hash_func = hash_func or hash

    def __getitem__(self, key):
        return self.contents[self._key(key)]

    def __setitem__(self, key, value):
        self.contents[self._key(key)] = value

    def __delitem__(self, key):
        del self.contents[self._key(key)]

    def __contains__(self, key):
        return self._key(key) in self.contents

    def __len__(self):
        return len(self.contents)

    def _key(self, key):
        return DictKey(self, key)

    def keys(self):
        return [k.key for k in self.contents.keys()]

    def values(self):
        return self.contents.values()

    def iteritems(self):
        for k, v in self.contents.iteritems():
            yield k.key, v

    def get(self, key, default):
        return self.contents.get(self._key(key), default)

    def pop(self, key, default=MARKER):
        if default is MARKER:
            return self.contents.pop(self._key(key))
        else:
            return self.contents.pop(self._key(key), default)

    def popitem(self):
        if not self:
            raise KeyError
        k, v = self.contents.popitem()
        return k.key, v

    def update(self, d):
        self.contents.update(d.contents)

    def clear(self):
        self.contents.clear()

    def copy(self):
        d = OrderedDict(self.eq_func, self.hash_func)
        for k, v in self.iteritems():
            d[k] = v
        return d
Example #9
0
def find_layers(pkts, pcap):
    packet = OrderedDict()
    count = 1
    try:
        for p in pkts:
            header = {
                "Buffer": {
                    "timestamp": datetime.datetime.fromtimestamp(p.time).strftime("%Y-%m-%d %H:%M:%S.%f"),
                    "packetnumber": count,
                    "pcapfile": pcap,
                }
            }
            packet.update(header)
            counter = 0
            while True:
                layer = p.getlayer(counter)
                if layer != None:
                    i = int(counter)
                    x = p[0][i].fields
                    t = exclude_layers(x, layer.name)
                    s = rename_layer(t, layer.name)
                    v = '{"' + layer.name.replace(".", "_") + "[" + str(i) + "]" + '":' + str(s) + "}"
                    s = eval(v)
                    packet.update(s)
                else:
                    break
                counter += 1
            count += 1
            yield packet
            packet.clear()
    except Exception as e:
        print RED + "Error within packet: " + str(
            count
        ) + ", on layer: " + layer.name + " . The full error was: " + str(e) + END
Example #10
0
def main():
    base_path = os.path.dirname(__file__)
    xml_file = os.path.join(base_path, "Survey-4.xml")
    csv_file = os.path.join(base_path, "Survey-4.csv")

    csv_headers_map = {"PIVA": "CUSTOMER_ID", "Email_Aziendale_Cliente": "ADDRESS", "Codice_Ticket": "SURVEY_ID"}

    csv_headers_required = ["CUSTOMER_ID", "ADDRESS", "SURVEY_ID", "LANGUAGE"]
    csv_headers_meta = None

    try:
        xml_tree = ET.parse(xml_file)
    except ET.ParseError as e:
        print(str.format("[#] parser error: {}", e))
        sys.exit(1)

    csv_headers = OrderedDict()
    for item in csv_headers_required:
        csv_headers[item] = None

    print(str.format("[*] csv_headers: {}", csv_headers))
    print(str.format("[*] csv_headers.key(): {}", csv_headers.keys()))
    print(str.format("[*] csv_headers.values(): {}", csv_headers.values()))
    print(str.format("[*] len(csv_headers): {}", len(csv_headers)))
    csv_headers.clear()
    print(str.format("[*] len(csv_headers): {}", len(csv_headers)))
Example #11
0
def get_available_skins(selected=None):
    """Returns a dictionary of skin name --> directory where
    "templates" and "media" subdirectories can be found.

    selected is a name of preferred skin
    if it's None, then information about all skins will be returned
    otherwise, only data about selected and default skins
    will be returned

    selected skin is guaranteed to be the first item in the dictionary
    """
    skins = OrderedDict()
    if hasattr(django_settings, "ASKBOT_EXTRA_SKINS_DIR"):
        skins.update(get_skins_from_dir(django_settings.ASKBOT_EXTRA_SKINS_DIR))

    if "default" in skins:
        raise ValueError('"default" is not an acceptable name for a custom skin')

    if selected in skins:
        selected_dir = skins[selected]
        skins.clear()
        skins[selected] = selected_dir
    elif selected == "default":
        skins = OrderedDict()
    elif selected:
        raise ValueError(
            "skin "
            + str(selected)
            + " not found, please check ASKBOT_EXTRA_SKINS_DIR setting "
            + "or in the corresponding directory"
        )

    # insert default as a last item
    skins["default"] = askbot.get_install_directory()
    return skins
Example #12
0
class LRUCache(Cache):
    """
    Implements a simple Least Recently Used Cache

    Very simple using collections.OrderedDict. The size can be changed during
    run-time
    """

    def __init__(self, size_limit):
        super(LRUCache, self).__init__()
        self._size_limit = size_limit
        self._cache = OrderedDict()

    @property
    def count(self):
        return len(self._cache), 0

    @property
    def size(self):
        return self.size_limit, 0

    @property
    def size_limit(self):
        return self._size_limit

    @size_limit.setter
    def size_limit(self, new_size):
        if new_size < self.size_limit:
            self._check_size_limit()

        self._size_limit = new_size

    def __iter__(self):
        return iter(self._cache)

    def __reversed__(self):
        return reversed(self._cache)

    def __getitem__(self, item):
        obj = self._cache.pop(item)
        self._cache[item] = obj
        return obj

    def __setitem__(self, key, value, **kwargs):
        self._cache[key] = value
        self._check_size_limit()

    def _check_size_limit(self):
        while len(self._cache) > self.size_limit:
            self._cache.popitem(last=False)

    def __contains__(self, item):
        return item in self._cache

    def clear(self):
        self._cache.clear()

    def __len__(self):
        return len(self._cache)
Example #13
0
class Classifier(object):
    """Parent factory for all classifier classes."""

    __metaclass__ = Factory

    def __init__(self):
        super(Classifier, self).__init__()
        self._pp = None  # setup preprocessor in the train method
        self.model = None
        self._clf = None
        self._classes = OrderedDict()

    def description(self):
        return

    def setParameters(self, params):
        assert isinstance(params, dict)
        self._clf.set_params(**params)

    def setupPreProcessor(self, features):
        self._pp = PreProcessor(features)

    def normalize(self, features):
        return self._pp(features)

    @property
    def classes(self):
        return self._classes

    def classByName(self, name):
        for class_ in self.classes.itervalues():
            if class_.name == name:
                return class_

        raise RuntimeError("No classes named %s" % name)

    def setClasses(self, classes, parent, panel):
        """Update the dictionary of the class definition and the list of
        the context menu actions."""

        self._classes.clear()
        self._classes.update(classes)

    @classmethod
    def classifiers(cls):
        return cls._classes.keys()

    def parameterWidget(self, parent):
        return QtWidgets.QWidget(self)

    def train(self, features):
        raise NotImplementedError

    def predict(self, features):
        return [UnClassified] * features.shape[0]

    def saveToHdf(self, file_, feature_names):
        raise NotImplementedError
Example #14
0
class Error_manager:
    def __init__(self):

        self.objec_error_dict = OrderedDict()
        self.Lines_With_Errors = []

    def log_error(self, object_name, verbose=True):

        self.objec_error_dict[object_name] = format_exc()

        if verbose:
            print "\n-- Error logged"

    def log_emLine_error(self, CodeName, Label):

        # Case for the first error log for a given object
        if Label not in self.objec_error_dict:
            LinesList = [[], []]

        # Case the objects has more lines with an error
        else:
            LinesList = self.objec_error_dict[CodeName]

        LinesList[0].append([Label, format_exc()])

        self.objec_error_dict[CodeName] = LinesList

    def display_errors(self, error_type="general", extra_verbose=False):

        if error_type == "general":

            if len(self.objec_error_dict) != 0:
                print "\n-These objects treatment produced a script error:\n"
                print self.objec_error_dict.keys(), "\n"
                for i in range(len(self.objec_error_dict)):
                    print "--Object", self.objec_error_dict.keys()[i],
                    print self.objec_error_dict.values()

                # We empty the dictionary as we don not need it anymore
                self.objec_error_dict.clear()

        elif error_type == "emLine measurement":

            if len(self.objec_error_dict) != 0:
                print "\n-These objects treatment produced a script error:\n"
                print self.objec_error_dict.keys(), "\n"

                for i in range(len(self.objec_error_dict)):
                    print "--Object", self.objec_error_dict.keys()[i]
                    print self.objec_error_dict.values()[i][0], "\n"

                if extra_verbose == True:
                    for i in range(len(self.objec_error_dict)):
                        print "--Object", self.objec_error_dict.keys()[i]
                        print "--Label", self.objec_error_dict.values()[i][0], "\n"
                        print "--Errors", self.objec_error_dict.values()[i][1], "\n"

        return " "
Example #15
0
 def clear(self):
     """
     Remove all keys.
     See: help(OrderedDict.clear)
     >>> T = pykov.Matrix({('A','B'): .3, ('A','A'): .7, ('B','A'): 1.})
     >>> T.clear()
     >>> T
     {}
     """
     OrderedDict.clear(self)
class CaselessDict(OrderedDict):
    """Case-insensitive dict. Values can be set and retrieved using keys of any case.
    Note that when iterating, the original case is returned for each key."""

    def __init__(self, *args):
        OrderedDict.__init__(self, {})  ## requirement for the empty {} here seems to be a python bug?
        self.keyMap = OrderedDict([(k.lower(), k) for k in OrderedDict.keys(self)])
        if len(args) == 0:
            return
        elif len(args) == 1 and isinstance(args[0], dict):
            for k in args[0]:
                self[k] = args[0][k]
        else:
            raise Exception("CaselessDict may only be instantiated with a single dict.")

    # def keys(self):
    # return self.keyMap.values()

    def __setitem__(self, key, val):
        kl = key.lower()
        if kl in self.keyMap:
            OrderedDict.__setitem__(self, self.keyMap[kl], val)
        else:
            OrderedDict.__setitem__(self, key, val)
            self.keyMap[kl] = key

    def __getitem__(self, key):
        kl = key.lower()
        if kl not in self.keyMap:
            raise KeyError(key)
        return OrderedDict.__getitem__(self, self.keyMap[kl])

    def __contains__(self, key):
        return key.lower() in self.keyMap

    def update(self, d):
        for k, v in d.iteritems():
            self[k] = v

    def copy(self):
        return CaselessDict(OrderedDict.copy(self))

    def __delitem__(self, key):
        kl = key.lower()
        if kl not in self.keyMap:
            raise KeyError(key)
        OrderedDict.__delitem__(self, self.keyMap[kl])
        del self.keyMap[kl]

    def __deepcopy__(self, memo):
        raise Exception("deepcopy not implemented")

    def clear(self):
        OrderedDict.clear(self)
        self.keyMap.clear()
Example #17
0
def getOutlinks(pmid, preferPmc=False):
    """ use NCBI eutils to get outlinks for a pmid as a dict provider -> url  """
    logging.debug("%s: Getting outlink from pubmed" % (pmid))
    url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id=%s&retmode=llinks&cmd=llinks" % pmid
    # logging.debug("getting %s" % url)
    # try:
    # except
    # logging.info(traceback.format_exc())
    # logging.info("Exception when downloading")
    # return None
    userAgent = "User-Agent: Mozilla (max@soe.ucsc.edu, http://text.soe.ucsc.edu)"

    # html = urllib2.urlopen(req)
    html = maxCommon.retryHttpRequest(url, userAgent=userAgent)

    if html == None:
        return None

    outlinks = OrderedDict()
    provider = False
    fullText = False
    isPmc = False

    for line in html:
        if line.find("<ObjUrl>") != -1:
            url = ""
            fullText = False
            origPublisher = False
        if line.find("Attribute") != -1:
            attribute = stripTag(line)
            if attribute.lower() == "full-text online" or attribute == "full-text pdf":
                fullText = True
        if line.find("<NameAbbr>") != -1 and fullText and origPublisher:
            db = stripTag(line)
            outlinks[db] = url
        if line.find("publishers/providers") != -1:
            origPublisher = True
        if line.find("<Provider>") != -1:
            provider = True
        if line.find("</Provider>") != -1:
            provider = False
        if line.find("<DbFrom>") != -1:
            db = stripTag(line)
        if line.find("<Url>") != -1 and not provider:
            url = line
            url = stripTag(url).replace("&amp;", "&")  # XX strange!
            url = stripTag(url).replace("&lt;", "<")
            url = stripTag(url).replace("&gt;", ">")
            if "www.ncbi.nlm.nih.gov" in url and "/pmc/" in url and preferPmc:
                # override all other links
                outlinks.clear()
                outlinks["pmc"] = url

    logging.debug("%s: Found outlinks %s" % (pmid, str(outlinks)))
    return outlinks
Example #18
0
 def clear(self):
     """ 
         clear all keys and keys in key_time_map
     """
     with self.lock:
         try:
             OrderedDict.clear(self)
             self.key_time_map.clear()
             return True
         except KeyError:
             return False
Example #19
0
class Tweak(object):  # {{{
    def __init__(self, name, doc, var_names, defaults, custom):
        translate = _
        self.name = translate(name)
        self.doc = doc.strip()
        if self.doc:
            self.doc = translate(self.doc)
        self.var_names = var_names
        if self.var_names:
            self.doc = u"%s: %s\n\n%s" % (_("ID"), self.var_names[0], self.doc)
        self.default_values = OrderedDict()
        for x in var_names:
            self.default_values[x] = defaults[x]
        self.custom_values = OrderedDict()
        for x in var_names:
            if x in custom:
                self.custom_values[x] = custom[x]

    def __str__(self):
        ans = ["#: " + self.name]
        for line in self.doc.splitlines():
            if line:
                ans.append("# " + line)
        for key, val in self.default_values.iteritems():
            val = self.custom_values.get(key, val)
            ans.append("%s = %r" % (key, val))
        ans = "\n".join(ans)
        if isinstance(ans, unicode):
            ans = ans.encode("utf-8")
        return ans

    def __cmp__(self, other):
        return -1 * cmp(self.is_customized, getattr(other, "is_customized", False))

    @property
    def is_customized(self):
        for x, val in self.default_values.iteritems():
            if self.custom_values.get(x, val) != val:
                return True
        return False

    @property
    def edit_text(self):
        ans = ["# %s" % self.name]
        for x, val in self.default_values.iteritems():
            val = self.custom_values.get(x, val)
            ans.append("%s = %r" % (x, val))
        return "\n\n".join(ans)

    def restore_to_default(self):
        self.custom_values.clear()

    def update(self, varmap):
        self.custom_values.update(varmap)
Example #20
0
class DeviceManager:
    """Handles creation and destruction of local device drivers and controller
    RPC clients."""

    def __init__(self, ddb, virtual_devices=dict()):
        self.ddb = ddb
        self.virtual_devices = virtual_devices
        self.active_devices = OrderedDict()

    def get_device_db(self):
        """Returns the full contents of the device database."""
        return self.ddb.get_device_db()

    def get_desc(self, name):
        desc = self.ddb.get(name)
        while isinstance(desc, str):
            # alias
            desc = self.ddb.get(desc)
        return desc

    def get(self, name):
        """Get the device driver or controller client corresponding to a
        device database entry."""
        if name in self.virtual_devices:
            return self.virtual_devices[name]
        if name in self.active_devices:
            return self.active_devices[name]
        else:
            try:
                desc = self.get_desc(name)
            except Exception as e:
                raise DeviceError("Failed to get description of device '{}'".format(name)) from e
            try:
                dev = _create_device(desc, self)
            except Exception as e:
                raise DeviceError("Failed to create device '{}'".format(name)) from e
            self.active_devices[name] = dev
            return dev

    def close_devices(self):
        """Closes all active devices, in the opposite order as they were
        requested."""
        for dev in reversed(list(self.active_devices.values())):
            try:
                if isinstance(dev, (Client, BestEffortClient)):
                    dev.close_rpc()
                elif hasattr(dev, "close"):
                    dev.close()
            except Exception as e:
                logger.warning("Exception %r when closing device %r", e, dev)
        self.active_devices.clear()
class Queue:
    def __init__(self, size=16):
        self.queue = OrderedDict([])
        self.size = size
        self.front = 0
        self.rear = 0

    def isEmpty(self):
        return (self.rear - self.front) == 0

    def isFull(self):
        if (self.rear - self.front) == self.size:
            return True
        else:
            return False

    def get(self, key):
        if key in self.queue:
            return self.queue[key]
        else:
            return False

    def add(self, key, value):
        if self.isFull():
            # raise Exception("QueueOverFlow")
            self.queue.popitem(last=False)
            self.queue[key] = value
            print self.queue.keys()
        else:
            self.queue[key] = value
            # print self.queue
            self.rear += 1

    def delete(self, key):
        if key in self.queue:
            # return

            a = self.queue.pop(key)
            # print self.queue.keys()
            # return a
            self.rear -= 1
            return a
        else:
            return False

    def delete_all(self):
        self.queue.clear()

    def show(self):
        pass
Example #22
0
    def test(self):
        tempDict = OrderedDict()
        for key, value in self.defect.items():
            #   print(key,value)
            #  temp = value.get(key)
            state = value.get("Defect Status", None)
            #   print('Test',key,state)
            if "Closed" in state:
                #  print('Closed')
                None
            elif "Fixed" in state:
                #  print('Fixed')
                None
            else:
                # print('ID',key,state)

                tempDict["Defect ID"] = key
                tempDict["Summary"] = value.get("Summary")
                tempDict["Severity"] = value.get("Severity")
                tempDict["Defect Status"] = value.get("Defect Status")
                tempDict["Status Whiteboard"] = value.get("Status Whiteboard")

                content = self.meeting.get(key, "empty")
                #     print ('Content',content)
                if not "empty" in content:
                    # print('Content',content)
                    for me_key, me_value in content.items():
                        if "Status Whiteboard" in me_key:
                            None
                        elif "Summary" in me_key:
                            None
                        elif "Severity" in me_key:
                            None
                        elif "ID" in me_key:
                            None
                        elif "Defect Status" in me_key:
                            None
                        elif "Priority" in me_key:
                            None
                        else:
                            tempDict[me_key] = me_value

                # print('HH',self.meeting.get(key,None))

                self.combined[key] = tempDict.copy()
                tempDict.clear()

        return self.combined
class BackgroundLogger(Logger):
    LOGGING_THREADS = logger.librarylogger.LOGGING_THREADS

    def __init__(self):
        self.lock = threading.RLock()
        self._messages = OrderedDict()

    def write(self, msg, level, html=False):
        with self.lock:
            thread = threading.currentThread().getName()
            if thread in self.LOGGING_THREADS:
                Logger.write(self, msg, level, html)
            else:
                message = BackgroundMessage(msg, level, html)
                self._messages.setdefault(thread, []).append(message)

    def log_background_messages(self, name=None):
        """Forwards messages logged on background to Robot Framework log.

        By default forwards all messages logged by all threads, but can be
        limited to a certain thread by passing thread's name as an argument.

        Logged messages are removed from the message storage.
        """
        with self.lock:
            if name:
                self._log_messages_by_thread(name)
            else:
                self._log_all_messages()

    def _log_messages_by_thread(self, name):
        for message in self._messages.pop(name, []):
            print(message.format())

    def _log_all_messages(self):
        for thread in list(self._messages):
            print("*HTML* <b>Messages by '%s'</b>" % thread)
            for message in self._messages.pop(thread):
                print(message.format())

    def reset_background_messages(self, name=None):
        with self.lock:
            if name:
                self._messages.pop(name)
            else:
                self._messages.clear()
Example #24
0
class DownloadCore:
    """"""

    def __init__(self):
        """"""
        self.active_downloads = {}
        self.queue_downloads = OrderedDict()
        self.complete_downloads = {}
        self.stopped_downloads = {}

    def reorder_queue(self, id_order_list):
        """
        TODO: only replace the order list in OrderedDict.
        """
        ordered_items_dict = OrderedDict()
        for id_item in id_order_list:
            try:
                ordered_items_dict[id_item] = self.queue_downloads[id_item]
            except KeyError:
                pass
        if len(self.queue_downloads) == len(ordered_items_dict):
            self.queue_downloads.clear()
            self.queue_downloads.update(ordered_items_dict)
        else:
            logger.warning("reorder_queue failed")

    # def reorder_queue(self, id_order_list):
    # """"""
    # self.reorder_list(self.queue_downloads, id_order_list)

    def reorder_list(self, download_list, id_order_list):
        """
        binding: los cambios en la 'copia' de la lista se reflejan en la lista original.
        """
        ordered_items_list = []
        items_dict = dict([(download_item.id, download_item) for download_item in download_list])
        for id_item in id_order_list:
            try:
                ordered_items_list.append(items_dict[id_item])
            except KeyError:
                pass
        if len(download_list) == len(ordered_items_list):
            download_list[:] = ordered_items_list  # reorder in-place
        else:
            logger.warning("reorder_list failed")
    def test_ordereddict(self):
        "AttrDict inherits all OrderedDict behaviour"

        items = [("x", 1), ("_y", 2), (3, 3)]
        ad = self.klass(items)
        od = OrderedDict(items)
        self.assertEqual(ad, od)
        self.assertEqual(ad.keys(), od.keys())
        self.assertEqual(ad.items(), od.items())

        ad.pop(items[0][0])
        od.pop(items[0][0])
        self.assertEqual(ad, od)

        ad.pop(items[1][0])
        od.pop(items[1][0])
        self.assertEqual(ad, od)

        ad["x"] = od["x"] = 1
        self.assertEqual(ad, od)

        ad.setdefault("x", 10)
        od.setdefault("x", 10)
        ad.setdefault("new", 10)
        od.setdefault("new", 10)
        self.assertEqual(ad, od)

        new_ad = ad.copy()
        new_od = od.copy()
        self.assertEqual(new_ad, new_od)
        self.assertEqual(type(new_ad), type(ad))

        ad.popitem()
        od.popitem()
        self.assertEqual(ad, od)

        ad.clear()
        od.clear()
        self.assertEqual(ad, od)

        ad = self.klass.fromkeys(range(10), 1)
        od = OrderedDict.fromkeys(range(10), 1)
        self.assertEqual(ad, od)
Example #26
0
def UpdateSplitSecurityIndex(mssqlDict, mongoDict):
    splitSecurityIndex = SplitSecurity(mssqlDict, mongoDict)
    splitDict = {"DAY": 200, "WEEK": 100, "MONTH": 50}
    for item in splitDict:
        # print item
        securityCodeList = splitSecurityIndex.GetSecurityCodeList(item)
        splitRangeList = splitSecurityIndex.SplitSecuritySubarea(securityCodeList, splitDict[item])
        num = 0
        for mongoHandle in splitSecurityIndex.mongoHandle:
            mongoHandle.drop("SplitIndex", item)
        document = OrderedDict()
        for splitRange in splitRangeList:
            document["_id"] = num
            document["Range"] = [splitRange["min"], splitRange["max"]]
            # print num,'min:',range['min'],', max:',range['max']
            for mongoHandle in splitSecurityIndex.mongoHandle:
                mongoHandle.insert("SplitIndex", item, document.copy())
            document.clear()
            num += 1
def find_layers(pkts, pcap, pcap_id, streamid):
    packet = OrderedDict()
    count = 1
    pcap_id = pcap_id.encode("utf-8")
    streamid = streamid.encode("utf-8")
    try:
        for p in pkts:
            header = {
                "Buffer": {
                    "timestamp": datetime.datetime.fromtimestamp(p.time).strftime("%Y-%m-%d %H:%M:%S.%f"),
                    "packetnumber": count,
                    "PCAP ID": pcap_id,
                    "pcapfile": pcap,
                    "StreamID": streamid,
                }
            }
            packet.update(header)
            counter = 0
            while True:
                layer = p.getlayer(counter)
                if layer != None:
                    i = int(counter)
                    x = p[0][i].fields
                    t = exclude_layers(x, layer.name)
                    s = rename_layer(t, layer.name)
                    v = '{"' + layer.name.replace(".", "_") + "[" + str(i) + "]" + '":' + str(s) + "}"
                    s = eval(v)
                    try:
                        del s["HTTP[3]"]
                        del s["HTTP[5]"]
                    except KeyError:
                        pass
                    packet.update(s)
                else:
                    break
                counter += 1
            count += 1
            yield packet
            packet.clear()
    except Exception as e:
        error_logging(str(e), "PacketParser")
        pass
class UniqueQueue(Queue.Queue):
    """Variant of Queue that only inserts unique items in the Queue."""

    def __init__(self, *args, **kwargs):
        """create the instance"""
        Queue.Queue.__init__(self, *args, **kwargs)
        self.logger = logging.getLogger("ubuntuone.SyncDaemon.HQ.Queue")

    def _init(self, maxsize):
        """Override the underlaying data initialization."""
        self.queue = OrderedDict()

    def _qsize(self, len=len):
        """The size of the queue."""
        return len(self.queue)

    def _put(self, item):
        """Custom _put that removes previous instances of this item."""
        key, value = item
        if key in self.queue:
            # we must delete it first, so the new one is added to the end
            del (self.queue[key])
            self.logger.debug("Previous item removed from the queue: %r", key)
        self.queue[key] = value

    def _get(self):
        """Custom _get that returns the first (key, value) pair."""
        return self.queue.popitem(last=False)

    def clear(self):
        """clear the internal queue and notify all blocked threads"""
        self.queue.clear()
        with self.all_tasks_done:
            self.unfinished_tasks = 0
            self.all_tasks_done.notifyAll()

    def __contains__(self, key):
        """Tell if a key is in the queue."""
        return key in self.queue
Example #29
0
class Metadator(Tk):
    def __init__(self):
        """
        Main window constructor
        Creates 1 frame and 2 labeled subframes
        """
        # first: the log
        # see: http://sametmax.com/ecrire-des-logs-en-python/
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)  # all errors will be get
        log_form = logging.Formatter("%(asctime)s || %(levelname)s || %(message)s")
        logfile = RotatingFileHandler("Metadator_LOG.log", "a", 5000000, 1)
        logfile.setLevel(logging.DEBUG)
        logfile.setFormatter(log_form)
        self.logger.addHandler(logfile)
        self.logger.info("\n\t ======== Metadator ========")  # first messages
        self.logger.info("Starting the UI")

        # checking the path to GDAL in the path
        if "GDAL_DATA" not in env.keys():
            try:
                gdal.SetConfigOption(str("GDAL_DATA"), str(path.abspath(r"data/gdal")))
            except:
                print(
                    "Oups! Something has gone wrong...\
                      see: https://github.com/Guts/Metadator/issues/21"
                )
        else:
            pass

        # basics settings
        Tk.__init__(self)  # constructor of parent graphic class
        self.title("Metadator {0}".format(MetadatorVersion))
        self.style = Style()  # more friendly windows style
        if opersys == "win32":
            self.logger.info("Op. system: {0}".format(platform.platform()))
            self.iconbitmap("Metadator.ico")  # windows icon
            self.uzer = env.get("USERNAME")
        elif opersys == "linux2":
            self.logger.info("Op. system: {0}".format(platform.platform()))
            self.uzer = env.get("USER")
            icon = Image("photo", file=r"data/img/metadator.gif")
            self.call("wm", "iconphoto", self._w, icon)
            self.minsize(580, 100)
            self.style.theme_use("clam")
        elif opersys == "darwin":
            self.logger.info("Op. system: {0}".format(platform.platform()))
            self.uzer = env.get("USER")
        else:
            self.logger.warning("Operating system not tested")
            self.logger.info("Op. system: {0}".format(platform.platform()))
        self.resizable(width=False, height=False)
        self.focus_force()

        self.logger.info("GDAL version: {}".format(gdal.__version__))

        # variables
        self.def_rep = ""  # folder to search for
        self.def_lang = "FR"  # language to start
        self.def_doc = IntVar()  # to export into Word
        self.def_xls = IntVar()  # to export into Excel 2003
        self.def_xml = IntVar()  # to export into ISO 19139
        self.def_cat = IntVar()  # to merge all output Word files
        self.def_odt = IntVar()  # to export into OpenDocumentText
        self.def_dict = IntVar()  # to make a dictionnary of data
        self.def_kass = IntVar()  # to handle field name case sensitive
        self.def_stat = IntVar()  # to active/disable stats fields
        self.li_pro = []  # list for profiles in language selected
        self.li_shp = []  # list for shapefiles path
        self.li_tab = []  # list for MapInfo tables path
        self.num_folders = 0  # number of folders explored
        self.today = strftime("%Y-%m-%d")  # date of the day
        self.dico_layer = OD()  # dictionary about layer properties
        self.dico_profil = OD()  # dictionary from profile selected
        self.dico_fields = OD()  # dictionary for fields information
        self.dico_rekur = OD()  # dictionary of recurring attributes
        self.dico_err = OD()  # errors list
        self.dico_help = OD()  # dictionary of help texts
        li_lang = [lg for lg in listdir(r"locale")]  # available languages
        self.blabla = OD()  # texts dictionary

        # GUI fonts
        ft_tit = tkFont.Font(family="Times", size=10, weight=tkFont.BOLD)

        # fillfulling
        self.load_settings()
        self.load_texts(self.def_lang)
        self.li_profiles(self.def_lang)
        self.li_rekurs(self.def_lang)
        self.recup_help(self.def_lang)

        # Tabs
        self.nb = Notebook(self)
        self.tab_globals = Frame(self.nb)  # tab_id = 0
        self.tab_options = Frame(self.nb)  # tab_id = 1
        self.tab_attribs = Frame(self.nb)  # tab_id = 2
        self.nb.add(self.tab_globals, text=self.blabla.get("gui_tab1"), padding=3)
        self.nb.add(self.tab_options, text=self.blabla.get("gui_tab2"), padding=3)
        self.nb.add(self.tab_attribs, text=self.blabla.get("gui_tab3"), padding=3)
        self.logger.info("UI created")

        ### Tab 1: global
        # Frames
        self.FrPath = Labelframe(self.tab_globals, name="main", text=self.blabla.get("tab1_fr1"))
        self.FrProg = Labelframe(self.tab_globals, name="progression", text=self.blabla.get("tab1_frprog"))
        ## Frame 1
        # target folder
        self.labtarg = Label(self.FrPath, text=self.blabla.get("tab1_path"))
        self.target = Entry(self.FrPath, width=25)
        self.browsetarg = Button(
            self.FrPath,  # browse button
            text=self.blabla.get("tab1_browse"),
            command=lambda: self.setpathtarg(),
            takefocus=True,
        )
        self.browsetarg.focus_force()  # force the focus on
        self.profil = Label(self.FrPath, text=self.blabla.get("tab1_prof"))
        # profiles switcher
        self.ddl_profil = Combobox(self.FrPath, values=self.li_pro, width=5)
        self.ddl_profil.current(0)
        self.ddl_profil.bind("<<ComboboxSelected>>", self.select_profil)
        # widgets placement
        self.labtarg.grid(row=1, column=1, columnspan=1, sticky=N + S + W + E, padx=2, pady=8)
        self.target.grid(row=1, column=2, columnspan=1, sticky=N + S + W + E, padx=2, pady=8)
        self.browsetarg.grid(row=1, column=3, sticky=N + S + W + E, padx=2, pady=8)
        self.profil.grid(row=2, column=1, sticky=N + S + W + E, padx=2, pady=8)
        self.ddl_profil.grid(row=2, column=2, sticky=W + E + N + S, columnspan=2, padx=2, pady=8)

        # tooltips
        InfoBulle(self.target, message=self.dico_help.get(30)[1])
        InfoBulle(self.browsetarg, message=self.dico_help.get(30)[1])
        InfoBulle(self.ddl_profil, message=self.dico_help.get(31)[1])

        ## Frame 2
        # variables
        self.status = StringVar(self.FrProg, "")
        # widgets
        self.prog_layers = Progressbar(self.FrProg, orient="horizontal")
        self.prog_fields = Progressbar(self.FrProg, orient="horizontal")
        # widgets placement
        Label(self.FrProg, textvariable=self.status, foreground="DodgerBlue").pack(expand=1)
        self.prog_layers.pack(expand=1, fill=X)

        # Frames placement
        self.FrPath.pack(expand=1, fill="both")
        self.FrProg.pack(expand=1, fill="both")

        ### Tab 2: options
        # Export options
        caz_doc = Checkbutton(
            self.tab_options,
            text="HTML / Word (.doc/.docx)",
            variable=self.def_doc,
            command=lambda: self.catalog_dependance(),
        )
        caz_xls = Checkbutton(self.tab_options, text="Excel 2003 (.xls)", variable=self.def_xls)
        caz_xml = Checkbutton(self.tab_options, text="XML (ISO 19139)", variable=self.def_xml)
        self.caz_cat = Checkbutton(self.tab_options, text=self.blabla.get("tab2_merge"), variable=self.def_cat)
        caz_odt = Checkbutton(self.tab_options, text="Open Document Text (.odt)", variable=self.def_odt)
        # widgets placement
        caz_doc.grid(row=1, column=0, sticky=N + S + W + E, padx=2, pady=2)
        self.caz_cat.grid(row=2, column=0, sticky=N + S + W + E, padx=2, pady=2)
        caz_xls.grid(row=1, column=1, sticky=N + S + W + E, padx=2, pady=2)
        caz_xml.grid(row=2, column=1, sticky=N + S + W + E, padx=2, pady=2)
        caz_odt.grid(row=3, column=1, sticky=N + S + W + E, padx=2, pady=2)
        # disabling the widgets which work only on Windows OS
        if opersys != "win32":
            self.logger.info("Disabling Windows reserved functions.")
            self.def_doc.set(0)
            self.def_cat.set(0)
            caz_doc.configure(state="disabled")
            self.caz_cat.configure(state="disabled")
        else:
            pass
        # make the catalog option depending on the Word option
        self.catalog_dependance()

        # tooltips
        InfoBulle(caz_doc, message=self.dico_help.get(33)[1], image=self.dico_help.get(33)[2])
        InfoBulle(caz_xls, message=self.dico_help.get(34)[1], image=self.dico_help.get(34)[2])
        InfoBulle(caz_xml, message=self.dico_help.get(35)[1], image=self.dico_help.get(35)[2])
        InfoBulle(caz_odt, message=self.dico_help.get(36)[1], image=self.dico_help.get(36)[2])
        InfoBulle(self.caz_cat, message=self.dico_help.get(37)[1], image=self.dico_help.get(37)[2])

        ### Tab 3: recurring attributes
        # Attribute selector
        self.lab_chps = Label(self.tab_attribs, text=self.blabla.get("tab3_sele"))
        self.ddl_attr = Combobox(self.tab_attribs, values=self.dico_rekur.keys())
        self.ddl_attr.bind("<<ComboboxSelected>>", self.edit_rekur)
        self.supr = Button(self.tab_attribs, text=self.blabla.get("tab3_supp"), command=self.del_rekur)
        # frame
        self.FrRekur = Labelframe(self.tab_attribs, name="attributes", text=self.blabla.get("tab3_tit"))
        # attribute settings
        self.tab3_LBnom = Label(self.FrRekur, text=self.blabla.get("tab3_nom"), state=DISABLED)
        self.tab3_ENnom = Entry(self.FrRekur, state=DISABLED)
        self.tab3_LBdesc = Label(self.FrRekur, text=self.blabla.get("tab3_desc"), state=DISABLED)
        self.tab3_TXdesc = Text(self.FrRekur, height=5, width=30, wrap=WORD, state=DISABLED)
        self.tab3_CBcass = Checkbutton(
            self.FrRekur, text=self.blabla.get("tab3_cass"), variable=self.def_kass, state=DISABLED
        )
        self.tab3_CBstat = Checkbutton(
            self.FrRekur, text=self.blabla.get("tab3_stat"), variable=self.def_stat, state=DISABLED
        )
        # Validation button
        self.save = Button(self.FrRekur, text=self.blabla.get("tab3_save"), command=self.save_rekur, state="disabled")

        # widgets placement
        self.lab_chps.grid(row=1, column=1, sticky=N + S + W, padx=2, pady=2)
        self.ddl_attr.grid(row=1, column=2, sticky=N + S + W + E, padx=2, pady=2)
        self.supr.grid(row=1, column=3, sticky=N + S + W + E, padx=2, pady=2)
        self.tab3_LBnom.grid(row=1, column=0, columnspan=1, sticky=N + S + W, padx=2, pady=2)
        self.tab3_ENnom.grid(row=1, column=1, columnspan=1, sticky=N + S + W + E, padx=2, pady=2)
        self.tab3_LBdesc.grid(row=2, column=0, columnspan=1, sticky=N + S + W + E, padx=2, pady=2)
        self.tab3_TXdesc.grid(row=2, column=1, columnspan=2, sticky=N + S + W + E, padx=2, pady=2)
        self.tab3_CBcass.grid(row=3, column=0, columnspan=1, sticky=N + S + W + E, padx=2, pady=2)
        self.tab3_CBstat.grid(row=3, column=1, columnspan=1, sticky=N + S + W + E, padx=2, pady=2)
        self.save.grid(row=5, column=0, columnspan=4, sticky=N + S + W + E, padx=2, pady=2)

        # Frame placement
        self.FrRekur.grid(row=2, column=1, columnspan=3, sticky=N + S + W + E, padx=2, pady=2)

        # tooltips
        InfoBulle(self.lab_chps, message=self.dico_help.get(38)[1])
        InfoBulle(self.ddl_attr, message=self.dico_help.get(39)[1])
        InfoBulle(self.supr, message=self.dico_help.get(40)[1])
        InfoBulle(self.tab3_CBcass, message=self.dico_help.get(41)[1])
        InfoBulle(self.tab3_CBstat, message=self.dico_help.get(42)[1])

        ## Main frame
        # Hola
        self.welcome = Label(self, text=self.blabla.get("hi") + self.uzer, font=ft_tit, foreground="red2")
        # Image
        self.icone = PhotoImage(master=self, file=r"data/img/metadator.gif")
        Label(self, image=self.icone).grid(row=2, column=0, padx=2, pady=2, sticky=N + S + W + E)
        # credits
        s = Style(self)
        s.configure("Kim.TButton", foreground="DodgerBlue", borderwidth=0, relief="flat")
        Button(
            self, text="by Julien M. (2015)", style="Kim.TButton", command=lambda: open_new("https://github.com/Guts")
        ).grid(row=3, padx=2, pady=2, sticky=W + E)
        # language switcher
        self.ddl_lang = Combobox(self, values=li_lang, width=5)
        self.ddl_lang.current(li_lang.index(self.def_lang))
        self.ddl_lang.bind("<<ComboboxSelected>>", self.change_lang)
        # Go go go button
        self.val = Button(self, text=self.blabla.get("tab1_go"), state="active", command=lambda: self.process())
        # Cancel button
        self.can = Button(self, text=self.blabla.get("gui_quit"), command=self.destroy)
        # widgets placement
        self.welcome.grid(row=0, column=0, columnspan=1, sticky=N + S + W + E, padx=2, pady=2)
        self.ddl_lang.grid(row=1, column=0, sticky=N, padx=2, pady=0)
        self.can.grid(row=4, column=0, sticky=N + S + W + E, padx=2, pady=2)
        self.val.grid(row=4, column=1, sticky=N + S + W + E, padx=2, pady=2)

        # tooltips
        InfoBulle(self.ddl_lang, message=self.dico_help.get(32)[1])

        ### Notebook placement
        self.nb.grid(row=0, rowspan=4, column=1, sticky=N + S + W + E)
        # keep updated list of profiles
        self.maj()

    def maj(self):
        """
        update the profiles dropdown list every second
        """
        try:
            self.li_profiles(self.ddl_lang.get())
            self.ddl_profil["values"] = self.li_pro
            self.after(1000, self.maj)
        except WindowsError:  # avoid an error occuring with browse button
            self.after(1000, self.maj)
            pass

    def alter_state(self, parent, new_state):
        """
        just a function to change easily  the state of  all children widgets
        of a parent class

        parent=Tkinter class with children (Frame, Labelframe, Tk, etc.)
        new_state=Tkinter keyword for widget state (ACTIVE, NORMAL, DISABLED)
        """
        for child in parent.winfo_children():
            child.configure(state=new_state)
        # end of function
        return parent, new_state

    def catalog_dependance(self):
        """ unselect the catalog option if the word option is unselected """
        if self.def_doc.get() == 0:
            self.def_cat.set(0)
            self.caz_cat.config(state="disabled")
        elif self.def_doc.get() == 1:
            self.caz_cat.config(state="normal")
        # end of function
        return

    def load_settings(self):
        """ load settings from last execution """
        confile = "options.ini"
        config = ConfigParser.RawConfigParser()
        config.read(confile)
        # basics
        self.def_lang = config.get("basics", "def_codelang")
        self.def_rep = config.get("basics", "def_rep")
        # export preferences
        self.def_doc.set(config.get("export_preferences", "def_word"))
        self.def_cat.set(config.get("export_preferences", "def_cat"))
        self.def_xls.set(config.get("export_preferences", "def_xls"))
        self.def_xml.set(config.get("export_preferences", "def_xml"))
        self.def_dict.set(config.get("export_preferences", "def_dict"))
        self.def_odt.set(config.get("export_preferences", "def_odt"))
        # log
        self.logger.info("Last options loaded")
        # End of function
        return config, self.def_rep, self.def_lang, self.def_doc

    def save_settings(self):
        """ save options in order to make the next execution easier """
        confile = "options.ini"
        config = ConfigParser.RawConfigParser()
        # add sections
        config.add_section("basics")
        config.add_section("export_preferences")
        # basics
        config.set("basics", "def_codelang", self.ddl_lang.get())
        config.set("basics", "def_rep", self.target.get())
        # export preferences
        config.set("export_preferences", "def_word", self.def_doc.get())
        config.set("export_preferences", "def_cat", self.def_cat.get())
        config.set("export_preferences", "def_xls", self.def_xls.get())
        config.set("export_preferences", "def_xml", self.def_xml.get())
        config.set("export_preferences", "def_dict", self.def_dict.get())
        config.set("export_preferences", "def_odt", self.def_odt.get())
        # Writing the configuration file
        with open(confile, "wb") as configfile:
            config.write(configfile)
        # End of function
        return config

    def change_lang(self, event):
        """ update the texts dictionary with the language selected """
        new_lang = event.widget.get()
        # change to the new language selected
        self.load_texts(new_lang)
        self.li_profiles(new_lang)
        self.li_rekurs(new_lang)
        self.ddl_profil.delete(0, END)
        self.ddl_profil.config(values=self.li_pro)
        self.ddl_profil.update()
        self.ddl_attr.config(values=self.dico_rekur.keys())
        self.recup_help(new_lang)
        # update widgets text
        # tab1
        self.nb.tab(0, text=self.blabla.get("gui_tab1"))
        self.welcome.config(text=self.blabla.get("hi") + self.uzer)
        self.can.config(text=self.blabla.get("gui_quit"))
        self.FrPath.config(text=self.blabla.get("tab1_fr1"))
        self.FrProg.config(text=self.blabla.get("tab1_frprog"))
        self.labtarg.config(text=self.blabla.get("tab1_path"))
        self.browsetarg.config(text=self.blabla.get("tab1_browse"))
        self.val.config(text=self.blabla.get("tab1_go"))
        self.profil.config(text=self.blabla.get("tab1_prof"))
        # tab2
        self.nb.tab(1, text=self.blabla.get("gui_tab2"))
        self.caz_cat.config(text=self.blabla.get("tab2_merge"))
        # tab3
        self.nb.tab(2, text=self.blabla.get("gui_tab3"))
        self.lab_chps.config(text=self.blabla.get("tab3_sele"))
        self.supr.config(text=self.blabla.get("tab3_supp"))
        self.FrRekur.config(text=self.blabla.get("tab3_tit"))
        self.tab3_LBnom.config(text=self.blabla.get("tab3_nom"))
        self.tab3_LBdesc.config(text=self.blabla.get("tab3_desc"))
        self.tab3_CBcass.config(text=self.blabla.get("tab3_cass"))
        self.tab3_CBstat.config(text=self.blabla.get("tab3_stat"))
        self.save.config(text=self.blabla.get("tab3_save"))

        # End of function
        return self.blabla

    def load_texts(self, lang="FR"):
        """
        Load texts according to the selected language
        """
        # clearing the text dictionary
        self.blabla.clear()
        # open xml cursor
        xml = ET.parse("locale/{0}/lang_{0}.xml".format(lang))
        # Looping and gathering texts from the xml file
        for elem in xml.getroot().getiterator():
            self.blabla[elem.tag] = elem.text
        # updating the GUI
        self.update()
        # en of function
        return self.blabla

    def setpathtarg(self):
        """ ...browse and insert the path of target folder """
        foldername = askdirectory(
            parent=self, initialdir=self.def_rep, mustexist=True, title=self.blabla.get("gui_cible")
        )
        # check if a folder has been choosen
        if foldername:
            try:
                self.target.delete(0, END)
                self.target.insert(0, foldername)
            except:
                info(title=self.blabla.get("nofolder"), message=self.blabla.get("nofolder"))
                return

        # count shapefiles and MapInfo files in a separated thread
        proc = threading.Thread(target=self.li_geofiles, args=(foldername,))
        proc.daemon = True
        proc.start()

        # end of function
        return foldername

    def li_geofiles(self, foldertarget):
        """ List shapefiles and MapInfo files (.tab, not .mid/mif) contained
        in the folders structure """
        # reseting global variables
        self.li_shp = []
        self.li_tab = []
        self.browsetarg.config(state=DISABLED)
        # Looping in folders structure
        self.status.set(self.blabla.get("tab1_prog1"))
        self.prog_layers.start()
        for root, dirs, files in walk(unicode(foldertarget)):
            self.num_folders = self.num_folders + len(dirs)
            for f in files:
                """ looking for files with geographic data """
                try:
                    unicode(path.join(root, f))
                    full_path = path.join(root, f)
                except UnicodeDecodeError:
                    full_path = path.join(root, f.decode("latin1"))
                # Looping on files contained
                if (
                    path.splitext(full_path.lower())[1].lower() == ".shp"
                    and (path.isfile("{0}.dbf".format(full_path[:-4])) or path.isfile("{0}.DBF".format(full_path[:-4])))
                    and (path.isfile("{0}.shx".format(full_path[:-4])) or path.isfile("{0}.SHX".format(full_path[:-4])))
                ):
                    """ listing compatible shapefiles """
                    # add complete path of shapefile
                    self.li_shp.append(full_path)
                elif (
                    path.splitext(full_path.lower())[1] == ".tab"
                    and (path.isfile(full_path[:-4] + ".dat") or path.isfile(full_path[:-4] + ".DAT"))
                    and (path.isfile(full_path[:-4] + ".map") or path.isfile(full_path[:-4] + ".MAP"))
                    and (path.isfile(full_path[:-4] + ".id") or path.isfile(full_path[:-4] + ".ID"))
                ):
                    """ listing MapInfo tables """
                    # add complete path of MapInfo file
                    self.li_tab.append(full_path)
        # stopping the progress bar
        self.prog_layers.stop()
        # Lists ordering and tupling
        self.li_shp.sort()
        self.li_shp = tuple(self.li_shp)
        self.li_tab.sort()
        self.li_tab = tuple(self.li_tab)
        # setting the label text and activing the buttons
        self.status.set(
            unicode(len(self.li_shp))
            + " shapefiles - "
            + unicode(len(self.li_tab))
            + " tables (MapInfo) - "
            + unicode(self.num_folders)
            + self.blabla.get("log_numfold")
        )
        self.browsetarg.config(state=ACTIVE)
        self.val.config(state=ACTIVE)
        # End of function
        return foldertarget, self.li_shp, self.li_tab

    def li_profiles(self, lang):
        """
        list profiles already existing
        """
        # reseting global variable
        self.li_pro = []
        # Looping in folders structure
        folder_profiles = path.join("locale/", lang + "/profiles/")
        self.li_pro = [lg[:-4] for lg in listdir(folder_profiles)]
        self.li_pro.append(self.blabla.get("tab1_new"))
        # End of function
        return folder_profiles, self.li_pro

    def li_rekurs(self, lang):
        """
        List recurring attributes that already exist in the selected language
        """
        # clearing the text dictionary
        self.dico_rekur.clear()
        champis = path.abspath(r"locale/{0}/champignons_{0}.xml".format(lang))
        xml = ET.parse(champis)
        # Looping and gathering texts from the xml file
        for elem in xml.findall("champ"):
            rek_name = elem.find("intitule").text
            rek_desc = elem.find("description").text
            rek_kass = elem.find("case").text
            rek_stat = elem.find("stats").text
            self.dico_rekur[rek_name] = rek_desc, rek_kass, rek_stat
        self.dico_rekur[self.blabla.get("tab3_new")] = "", 0, 0
        # updating the GUI
        self.update()
        # End of function
        return self.dico_rekur

    def edit_rekur(self, event):
        """
        preparing the form to edit a recurring attribute
        """
        rekur = event.widget.get()
        # deactivate the selector
        self.ddl_attr.config(state=DISABLED)
        # activate the form
        self.alter_state(self.FrRekur, NORMAL)
        # change to the new language selected
        self.tab3_ENnom.insert(0, rekur)
        self.tab3_TXdesc.insert(1.0, self.dico_rekur.get(rekur)[0])
        self.def_kass.set(self.dico_rekur.get(rekur)[1])
        self.def_stat.set(self.dico_rekur.get(rekur)[2])
        # End of function
        return self.dico_rekur

    def save_rekur(self):
        """ save the recurring attribute edited """
        # check if the attribute already exists
        if self.tab3_ENnom.get() in self.dico_rekur:
            if not askyesno(title=self.blabla.get("tab3_alert_exist1"), message=self.blabla.get("tab3_alert_exist2")):
                return
            else:
                pass
        else:
            pass

        # save
        self.dico_rekur[self.tab3_ENnom.get()] = (
            self.tab3_TXdesc.get(1.0, END).rstrip(),
            self.def_kass.get(),
            self.def_stat.get(),
        )
        # reset the form
        self.tab3_ENnom.delete(0, END)
        self.tab3_TXdesc.delete(1.0, END)
        self.def_kass.set(0)
        self.def_stat.set(0)
        # deactivate the form
        self.alter_state(self.FrRekur, DISABLED)
        # updating the dropdown list
        self.ddl_attr.config(state=NORMAL)
        self.ddl_attr.delete(0, END)
        self.ddl_attr["values"] = self.dico_rekur.keys()

        # End of function
        return self.dico_rekur

    def del_rekur(self):
        """ delete the selected recurring attribute """
        # reactivate the selector
        self.ddl_attr.config(state=ACTIVE)
        self.dico_rekur.pop(self.ddl_attr.get())

        self.ddl_attr.delete(0, END)
        self.ddl_attr["values"] = self.dico_rekur.keys()
        # reset the form
        self.tab3_ENnom.delete(0, END)
        self.tab3_TXdesc.delete(1.0, END)
        self.def_kass.set(0)
        self.def_stat.set(0)
        # deactivate the form
        self.alter_state(self.FrRekur, DISABLED)

        # End of function
        return self.dico_rekur

    def saveas_rekurs(self, lang):
        """ save the recurring fields into the file dedicated """
        rekur = ET.Element("champs")
        xml_path = r"locale/{0}/champignons_{0}.xml".format(lang)
        self.dico_rekur.pop(self.blabla.get("tab3_new"))
        with open(xml_path, "w") as champis:
            for elem in self.dico_rekur.keys():
                rek = ET.SubElement(rekur, "champ")
                # name of recurring attribute
                rek_name = ET.SubElement(rek, "intitule")
                rek_name.text = elem
                # description of recurring attribute
                rek_desc = ET.SubElement(rek, "description")
                rek_desc.text = self.dico_rekur.get(elem)[0]
                # stats option of recurring attribute
                rek_stats = ET.SubElement(rek, "stats")
                rek_stats.text = unicode(self.dico_rekur.get(elem)[1])
                # case sensitive option of recurring attribute
                rek_case = ET.SubElement(rek, "case")
                rek_case.text = unicode(self.dico_rekur.get(elem)[2])

        # creating the xml tree
        out_rekurs = ET.ElementTree(rekur)
        # saving it
        out_rekurs.write(xml_path, encoding="utf-8", xml_declaration='version="1.0"', method="xml")

        # End of function
        return self.dico_rekur

    def select_profil(self, event):
        """ when a profile is selected... """
        profsel = event.widget.get()
        # if user wants to use an existing profile or create a new one
        if profsel == self.blabla.get("tab1_new"):
            self.val.config(text=self.blabla.get("tab1_crprofil"))
        else:
            self.val.config(text=self.blabla.get("tab1_go"))

        # end of function
        return self.val

    def recup_profil(self, lang):
        """ get the information from the profile selected """
        # clearing the profile dictionary
        self.dico_profil.clear()
        # specific path to profile file
        path_profile = path.join("locale/{0}/profiles/{1}.xml".format(lang, self.ddl_profil.get()))
        with open(path_profile, "r") as profile:
            # open xml parser
            xml = ET.parse(profile)
            # basic informations
            self.dico_profil["description"] = xml.find("description").text
            self.dico_profil["sources"] = xml.find("sources").text
            self.dico_profil["url"] = xml.find("url").text
            self.dico_profil["url_label"] = xml.find("url_label").text
            self.dico_profil["diffusion"] = xml.find("diffusion").text
            # data language
            lang_data = xml.find("lang_data")
            self.dico_profil["lang_data"] = lang_data.find("name").text
            # metadata language
            lang_metad = xml.find("lang_metad")
            self.dico_profil["lang_md"] = lang_metad.find("name").text
            # diffusion constraints
            diff = xml.find("diffusion")
            self.dico_profil["diffusion"] = diff.find("name").text
            # update rythm
            rythm = xml.find("rythm")
            self.dico_profil["rythm"] = rythm.find("name").text
            # INSPIRE themes
            themes = xml.find("themesinspire")
            li_themesinspire = [theme.find("name").text for theme in themes.findall("theme")]
            self.dico_profil["themesinspire"] = li_themesinspire
            # custom keywords
            keywords = xml.find("keywords")
            li_keywords = [keyword.find("name").text for keyword in keywords.findall("keyword")]
            self.dico_profil["keywords"] = li_keywords
            # places keywords
            geokeywords = xml.find("geokeywords")
            li_geokeywords = [geokeyword.find("name").text for geokeyword in geokeywords.findall("geokeyword")]
            self.dico_profil["geokeywords"] = li_geokeywords
            # contacts
            contacts = xml.find("contacts")
            # point of contact
            cont = contacts.find("pointdecontact")
            self.dico_profil["cont_name"] = cont.find("name").text
            self.dico_profil["cont_orga"] = cont.find("org").text
            self.dico_profil["cont_mail"] = cont.find("mail").text
            self.dico_profil["cont_role"] = cont.find("role").text
            self.dico_profil["cont_func"] = cont.find("func")[0].text
            self.dico_profil["cont_street"] = cont.find("street").text
            self.dico_profil["cont_city"] = cont.find("city").text
            self.dico_profil["cont_cp"] = cont.find("cp").text
            self.dico_profil["cont_country"] = cont.find("country").text
            self.dico_profil["cont_phone"] = cont.find("tel").text
            # second contact (responsable, etc.)
            resp = contacts.find("second_contact")
            self.dico_profil["resp_name"] = resp.find("name").text
            self.dico_profil["resp_orga"] = resp.find("org").text
            self.dico_profil["resp_mail"] = resp.find("mail").text
            self.dico_profil["resp_role"] = resp.find("role").text
            self.dico_profil["resp_func"] = resp.find("func")[0].text
            self.dico_profil["resp_street"] = resp.find("street").text
            self.dico_profil["resp_city"] = resp.find("city").text
            self.dico_profil["resp_cp"] = resp.find("cp").text
            self.dico_profil["resp_country"] = resp.find("country").text
            self.dico_profil["resp_phone"] = resp.find("tel").text
        # End of function
        return self.dico_profil

    def recup_help(self, lang):
        """ get the help texts """
        # specific path to xml file
        path_help = "locale/%s/help_%s.xml" % (lang, lang)
        # reading and parsing the xml
        with open(path_help, "r") as source:
            xml = ET.parse(source)  # xml cursor
            for tooltip in xml.findall("tooltip"):
                idu = tooltip.find("id").text
                ref = tooltip.find("ref").text
                txt = tooltip.find("txt").text
                img = tooltip.find("image").text
                doc = tooltip.find("doc").text
                # fillfulling the INSPIRE dictionary
                self.dico_help[int(idu)] = ref, txt, img, doc
        # End of function
        return self.dico_help

    def process(self):
        """ launch the different processes """
        # display the main tab
        self.nb.select(0)
        # check option selected: process or create a new profile
        if self.ddl_profil.get() == self.blabla.get("tab1_new"):
            # launching the profile form
            self.logger.info("Creation of a new profile")
            tr_profile = threading.Thread(
                target=NewProfile, args=(self.blabla, self.ddl_lang.get(), self.dico_help, self.li_pro)
            )
            tr_profile.daemon = True
            tr_profile.run()
            # NewProfile(self.blabla, self.ddl_lang.get(), self.li_pro)
            self.li_profiles(self.ddl_lang.get())  # updating the dropdow list
            self.ddl_profil["values"] = self.li_pro
            return
        # check if the target folder has been selected
        if self.target.get() == "":
            info(title=self.blabla.get("info_blanktarget1"), message=self.blabla.get("info_blanktarget2"))
            return
        # check if a profile has been selected
        if self.ddl_profil.get() == "":
            info(title=self.blabla.get("info_blankprofile1"), message=self.blabla.get("info_blankprofile2"))
            return
        # disabling others GUI parts
        self.tab_globals.focus_force()
        self.alter_state(self.FrPath, DISABLED)

        # check if there are some layers into the folder structure
        if len(self.li_shp) + len(self.li_tab) == 0:
            self.logger.warning("No geofiles found in the folder structure")
            self.status.set(self.blabla.get("log_nodata"))
            return
        # specific variables
        dest = path.join(self.target.get(), "metadator")
        if not path.isdir(dest):  # test if folder already exists
            mkdir(dest, 0777)  # if not, we create it
        # getting profile informations
        self.recup_profil(self.ddl_lang.get())
        # saving options in a separated thread
        tr_options = threading.Thread(target=self.save_settings)
        tr_options.daemon = True
        tr_options.start()
        self.logger.info("Current options saved")
        # saving recurring fiels in a separated thread
        tr_rekurs = threading.Thread(target=self.saveas_rekurs, args=(self.ddl_lang.get(),))
        tr_rekurs.daemon = True
        tr_rekurs.start()
        # configuring the progression bar
        self.prog_layers["maximum"] = len(self.li_shp) + len(self.li_tab)
        self.prog_layers["value"]
        # Processing the shapefiles
        self.logger.info("\tStart processing the files")
        for shp in self.li_shp:
            """ looping on shapefiles list """
            self.logger.info("Processing: %s" % path.basename(shp))
            self.status.set(path.basename(shp))
            # reset recipient data
            self.dico_layer.clear()
            self.dico_fields.clear()
            # getting separated process threads
            Read_SHP(shp, self.dico_layer, self.dico_fields, "shape", self.blabla)
            # checking layer error
            if self.dico_layer.get("error"):
                # increment the progress bar
                self.prog_layers["value"] = self.prog_layers["value"] + 1
                self.update()
                self.logger.warning("This shape has an issue: %s" % shp)
                continue
            # getting fields statistics only if needed
            if self.def_doc.get() == 1 or self.def_xls.get() == 1 or self.def_odt.get() == 1:
                StatsFields(shp, self.dico_fields, self.dico_rekur, self.blabla)
            # export according to options selected
            if self.def_doc.get() == 1:
                ExportToHTML(dest, self.dico_layer, self.dico_fields, self.dico_profil, self.dico_rekur, self.blabla)
                html_path = path.join(dest, "{0}_MD.html".format(self.dico_layer["name"][:-4]))
                ExportToDocX(html_path, dest)
            if self.def_xls.get() == 1:
                ExportToXLS(dest, self.dico_layer, self.dico_fields, self.dico_profil, self.dico_rekur, self.blabla)
            if self.def_xml.get() == 1:
                ExportToXML(dest, self.dico_layer, self.dico_profil, "", self.blabla, 1, 0)
            if self.def_odt.get() == 1:
                ExportToODT(dest, self.dico_layer, self.dico_fields, self.dico_profil, self.dico_rekur, self.blabla)
            # increment the progress bar
            self.prog_layers["value"] = self.prog_layers["value"] + 1
            self.update()

        # Processing the MapInfo tables
        for tab in self.li_tab:
            """ looping on MapInfo tables list """
            self.logger.info("Processing: %s" % path.basename(tab))
            self.status.set(path.basename(tab))
            # reset recipient data
            self.dico_layer.clear()
            self.dico_fields.clear()
            # getting the informations
            Read_TAB(tab, self.dico_layer, self.dico_fields, "table", self.blabla)
            # checking layer error
            if self.dico_layer.get("error"):
                self.logger.warning("This MapInfo table has an issue: %s" % tab)
                # increment the progress bar
                self.prog_layers["value"] = self.prog_layers["value"] + 1
                self.update()
                continue
            # getting fields statistics only if needed
            if self.def_doc.get() == 1 or self.def_xls.get() == 1 or self.def_odt.get() == 1:
                StatsFields(tab, self.dico_fields, self.dico_rekur, self.blabla)
            # export according to options selected
            if self.def_doc.get() == 1:
                ExportToHTML(dest, self.dico_layer, self.dico_fields, self.dico_profil, self.dico_rekur, self.blabla)
                html_path = path.join(dest, "{0}_MD.html".format(self.dico_layer["name"][:-4]))
                ExportToDocX(html_path, dest)
            if self.def_xls.get() == 1:
                ExportToXLS(dest, self.dico_layer, self.dico_fields, self.dico_profil, self.dico_rekur, self.blabla)
            if self.def_xml.get() == 1:
                ExportToXML(dest, self.dico_layer, self.dico_profil, "", self.blabla, 1, 0)
            if self.def_odt.get() == 1:
                ExportToODT(dest, self.dico_layer, self.dico_fields, self.dico_profil, self.dico_rekur, self.blabla)
            # increment the progress bar
            self.prog_layers["value"] = self.prog_layers["value"] + 1
            self.update()

        # Word catalog export
        if self.def_doc.get() == 1 and self.def_cat.get() == 1:
            self.status.set(self.blabla.get("info_cat"))
            self.update()
            DocxMerger(dest, "00_Metadator_Catalog", "metadator_")
        else:
            pass

        # final message
        # msg = self.blabla.get('info_end2') + self.blabla.get('info_end3')
        # info(title=self.blabla.get('info_end'), message=msg)
        # opening the destination folder
        self.open_dir_file(dest)
        # cleaning up
        logging.info("Hurray! It worked! All seem to have been fine!")
        self.destroy()
        # end of function
        return

    def open_dir_file(self, target):
        """
        Open a file or a directory in the explorer of the operating system
        http://sametmax.com/ouvrir-un-fichier-avec-le-bon-programme-en-python
        """
        # check if the file or the directory exists
        if not path.exists(target):
            raise IOError("No such file: {0}".format(target))

        # check the read permission
        if not access(target, R_OK):
            raise IOError("Cannot access file: {0}".format(target))

        # open the directory or the file according to the os
        if opersys == "win32":  # Windows
            proc = startfile(target)

        elif opersys.startswith("linux"):  # Linux:
            proc = subprocess.Popen(["xdg-open", target], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        elif opersys == "darwin":  # Mac:
            proc = subprocess.Popen(["open", "--", target], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        else:
            raise NotImplementedError("Your `%s` isn't a supported operating system`." % opersys)

        # end of function
        return proc
Example #30
0
class LRUCache(object):
    """Least-recently-used cache with dict-like usage"""

    # The idea for using an OrderedDict comes from Kun Xi:
    # http://www.kunxi.org/blog/2014/05/lru-cache-in-python/

    _SENTINEL = object()

    def __init__(self, capacity=2048):
        self._capacity = capacity
        self._cache = OrderedDict()
        self._hits = 0
        self._misses = 0

    def __repr__(self):
        hitrate = 1.0
        missrate = 0.0
        accesses = float(self._hits + self._misses)
        if accesses > 0:
            hitrate = self._hits / accesses
            missrate = self._misses / accesses
        return "<LRUCache c: %d/%d h: %.0f%% m: %.0f%%>" % (
            len(self._cache),
            self._capacity,
            hitrate * 100,
            missrate * 100,
        )

    def clear(self):
        self._cache.clear()
        self._hits = 0
        self._misses = 0

    def __len__(self):
        return len(self._cache)

    def __contains__(self, key):
        return key in self._cache

    def __getitem__(self, key):
        item = self.get(key, self._SENTINEL)
        if item is self._SENTINEL:
            raise KeyError
        return item

    def get(self, key, default=None):
        try:
            item = self._cache.pop(key)
            self._cache[key] = item
            self._hits += 1
            return item
        except KeyError:
            self._misses += 1
            return default

    def __setitem__(self, key, item):
        try:
            self._cache.pop(key)
        except KeyError:
            while len(self._cache) >= self._capacity:
                self._cache.popitem(last=False)
        self._cache[key] = item