def __init__(self, namespace=None):
     self.iptables = iptables_manager.IptablesManager(use_ipv6=ipv6_utils.is_enabled(), namespace=namespace)
     # TODO(majopela, shihanzhang): refactor out ipset to a separate
     # driver composed over this one
     self.ipset = ipset_manager.IpsetManager(namespace=namespace)
     self.ipconntrack = ip_conntrack.IpConntrackManager(self.get_device_zone, namespace=namespace)
     self._populate_initial_zone_map()
     # list of port which has security group
     self.filtered_ports = {}
     self.unfiltered_ports = {}
     self._add_fallback_chain_v4v6()
     self._defer_apply = False
     self._pre_defer_filtered_ports = None
     self._pre_defer_unfiltered_ports = None
     # List of security group rules for ports residing on this host
     self.sg_rules = {}
     self.pre_sg_rules = None
     # List of security group member ips for ports residing on this host
     self.sg_members = collections.defaultdict(lambda: collections.defaultdict(list))
     self.pre_sg_members = None
     self.enable_ipset = cfg.CONF.SECURITYGROUP.enable_ipset
     self._enabled_netfilter_for_bridges = False
     self.updated_rule_sg_ids = set()
     self.updated_sg_members = set()
     self.devices_with_updated_sg_members = collections.defaultdict(list)
Example #2
1
 def __init__(self):
     # keep around a queue for each switch of pending openflow messages waiting to
     # arrive at the switches.
     # { pending receive -> [(connection, pending ofp)_1, (connection, pending ofp)_2, ...] }
     self.pendingreceive2conn_messages = defaultdict(list)
     # { pending send -> [(connection, pending ofp)_1, (connection, pending ofp)_2, ...] }
     self.pendingsend2conn_messages = defaultdict(list)
Example #3
1
def success_view(request):
    from collections import defaultdict
    import nltk

    login = authenticated_userid(request)
    if login:
        user = DBSession.query(User).filter(User.login == login).first()
        pages = DBSession.query(Page).all()
        kappa = defaultdict(int)
        annotators = defaultdict(int)

        for page in pages:
            annotators[page.id], = (
                DBSession.query(func.count(distinct(Annotation.random_identifier)))
                .filter(Annotation.page_id == page.id)
                .first()
            )
            print "annotators:{}".format(annotators)
            annotations = DBSession.query(Annotation).filter(Annotation.page_id == page.id).all()

            try:
                data = [
                    (annotation.random_identifier, annotation.data_id, annotation.result) for annotation in annotations
                ]
                agreement = nltk.metrics.agreement.AnnotationTask(data=data)
                kappa[page.id] = agreement.kappa()
            except ZeroDivisionError, e:
                print data, e
        return {"user": user, "pages": pages, "kappa": kappa}
def serviceAuthorizations(request):
    authentication = platform_manager.authenticate(request)
    if "error" in authentication:
        return HttpResponse(authentication["response"])

    user = authentication["user"]

    return_v = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(str))))
    applications = application_manager.getApplications()
    for application in applications:
        scopes = application_manager.getApplicationScopes(application)
        return_v[application._id]["name"] = application.name
        return_v[application._id]["owner"] = application.user.get_full_name()
        return_v[application._id]["description"] = application.description
        connectors = set()

        for scope in scopes:
            return_v[application._id]["scopes"][scope.scope]["description"] = scope.description
            connector = scope.connector
            connectors.add(connector)
            return_v[application._id]["scopes"][scope.scope]["status"] = getAuthorizationStatus(
                user, application, scope
            )
            return_v[application._id]["scopes"][scope.scope]["connector"] = connector.name
        for connector in connectors:
            uri = createUri(user, application, connector)
            return_v[application._id]["connectors"][connector.name]["grant_uri"] = uri["grant_uri"]
            return_v[application._id]["connectors"][connector.name]["revoke_uri"] = uri["revoke_uri"]

    return HttpResponse(json.dumps({"service": dict(return_v)}))
def make_examples_simple(data_dir, n_users, negative_examples_per_user=10):
    G = snap.LoadEdgeList(snap.PUNGraph, data_dir + "graph.txt", 0, 1)
    new_edges = defaultdict(dict)
    with open(data_dir + "new_edges.txt") as f:
        for line in f:
            u, b = map(int, line.split())
            new_edges[u][b] = 1

    businesses = map(int, util.load_json(data_dir + "business.json").keys())
    examples = defaultdict(dict)
    users = random.sample([NI.GetId() for NI in G.Nodes()], n_users)
    for u in users:
        examples[u] = new_edges[u]
        for i in range(negative_examples_per_user):
            b = random.choice(businesses)
            examples[u][b] = 0

    p, n = 0, 0
    for u in examples:
        for b in examples[u]:
            p += examples[u][b]
            n += 1 - examples[u][b]
    print "Positive:", p
    print "Negative:", n
    print "Data skew:", p / float(p + n)
    print "Sampling rate:", negative_examples_per_user / float(len(businesses))

    print "Writing examples..."
    util.write_json(examples, data_dir + "examples_simple.json")
Example #6
1
    def muster_basis_options(self):
        text = ""
        lowername = self.method.lower()
        options = defaultdict(lambda: defaultdict(dict))

        options["BASIS"]["ORBITAL"]["value"] = self.basis

        if self.method in ["ccsd(t)-f12-optri"]:
            if self.basis == "cc-pvdz-f12":
                options["BASIS"]["JKFIT"]["value"] = "aug-cc-pvtz/jkfit"
                options["BASIS"]["JKFITC"]["value"] = self.basis + "/optri"
                options["BASIS"]["MP2FIT"]["value"] = "aug-cc-pvtz/mp2fit"
        elif (
            ("df-" in self.method)
            or ("f12" in self.method)
            or (self.method in ["mp2c", "dft-sapt", "dft-sapt-pbe0acalda"])
        ):
            if self.unaugbasis and self.auxbasis:
                options["BASIS"]["JKFIT"]["value"] = self.auxbasis + "/jkfit"
                options["BASIS"]["JKFITB"]["value"] = self.unaugbasis + "/jkfit"
                options["BASIS"]["MP2FIT"]["value"] = self.auxbasis + "/mp2fit"
                options["BASIS"]["DFLHF"]["value"] = self.auxbasis + "/jkfit"
            else:
                raise ValidationError("""Auxiliary basis not predictable from orbital basis '%s'""" % (self.basis))
        return text, options
Example #7
0
def diagnose_chromosome_ellipsoid(samples_to_test, ellipsoid, list_of_amplicons_to_test, qChisq, num_of_accepted):
    """

    :param samples_to_test: coverages in samples and amplicons, dict {sample name : {ampl name : coverage} }
    :param ellipsoid: pairs for each amplicon, (estimation of mean, estimation of standard deviation)
    :param list_of_amplicons_to_test: list of amplicons from one chromosome without low covered amplicons
    :param qChisq: corresponding chi square quantile
    :param num_of_accepted: number of amplicons that will be taken into account
    :return: dict of lists with normal or not coverages inside one chromosome ( 1 = normal, 0 = irregular);
             average robust residuals for calculation of ARV
    """
    normal_or_not = defaultdict(int)
    statistic_for_sample_and_chromosome = defaultdict(list)
    avtc_residuals_for_amplicons = defaultdict(list)

    for sample, coverages_of_amplicons in samples_to_test.iteritems():
        for ampl in list_of_amplicons_to_test:
            distance_to_mean = coverages_of_amplicons[ampl] - ellipsoid[ampl][0]
            dist = (distance_to_mean ** 2) / (ellipsoid[ampl][1])
            statistic_for_sample_and_chromosome[sample].append(dist)
            avtc_residuals_for_amplicons[ampl].append(distance_to_mean ** 2)
    for sample, statistic_values in statistic_for_sample_and_chromosome.iteritems():
        if sum(sorted(statistic_values)[:num_of_accepted]) < qChisq:
            normal_or_not[sample] = 1
        else:
            normal_or_not[sample] = 0
    return normal_or_not, avtc_residuals_for_amplicons
Example #8
0
File: core.py Project: keis/uzbl
    def __init__(self, parent, proto, print_events=False):
        proto.target = self
        self.print_events = print_events
        self.parent = parent
        self.proto = proto
        self.time = time.time()
        self.pid = None
        self.name = None

        # Flag if the instance has raised the INSTANCE_START event.
        self.instance_start = False

        # Use name "unknown" until name is discovered.
        self.logger = logging.getLogger("uzbl-instance[]")

        # Plugin instances
        self._plugin_instances = []
        self.plugins = {}

        # Track plugin event handlers
        self.handlers = defaultdict(list)
        self.request_handlers = defaultdict(list)

        # Internal vars
        self._depth = 0
        self._buffer = ""
Example #9
0
 def _clear(self):
     self.__displayInfo = {}
     self.__displaySettings = {}
     self.__topLevels = defaultdict(set)
     self.__topItems = defaultdict(set)
     self.__nextLevels = defaultdict(dict)
     self.__unlockPrices = defaultdict(dict)
Example #10
0
def fuzzy_find(value, options, stopwords=[]):
    """Try find the option nearest to the value.
    
        >>> fuzzy_find("O'Reilly", ["O'Reilly Inc", "Addison-Wesley"])
        "O'Reilly Inc"
    """
    if not options:
        return value

    rx = web.re_compile("[-_\.&, ]+")

    # build word frequency
    d = defaultdict(list)
    for option in options:
        for t in rx.split(option):
            d[t].append(option)

    # find score for each option
    score = defaultdict(lambda: 0)
    for t in rx.split(value):
        if t.lower() in stopwords:
            continue
        for option in d[t]:
            score[option] += 1

    # take the option with maximum score
    return max(options, key=score.__getitem__)
Example #11
0
    def as_coefficients_dict(a):
        """Return a dictionary mapping terms to their Rational coefficient.
        Since the dictionary is a defaultdict, inquiries about terms which
        were not present will return a coefficient of 0. If an expression is
        not an Add it is considered to have a single term.

        Examples
        ========

        >>> from sympy.abc import a, x
        >>> (3*x + a*x + 4).as_coefficients_dict()
        {1: 4, x: 3, a*x: 1}
        >>> _[a]
        0
        >>> (3*a*x).as_coefficients_dict()
        {a*x: 3}
        """

        d = defaultdict(list)
        for ai in a.args:
            c, m = ai.as_coeff_Mul()
            d[m].append(c)
        for k, v in d.items():
            if len(v) == 1:
                d[k] = v[0]
            else:
                d[k] = Add(*v)
        di = defaultdict(int)
        di.update(d)
        return di
def getDefLib():
    defense_stats = []
    steam_names = []
    defense_parameters = []
    Def_Lib = defaultdict(lambda: defaultdict(dict))

    defense = open("Defense.csv.csv", "rU")
    defense_read = csv.reader(defense, dialect=csv.excel_tab)
    for param in defense_read.next()[0].split(","):
        defense_parameters.append(param)

    Week = 1
    for line in defense:
        if "Week" not in line:
            single_params = line.split(",")
            team_name = abbrvs[single_params[0].split(". ")[1]]

            Def_Lib[Week][team_name]["Sacks"] = single_params[2]
            Def_Lib[Week][team_name]["Recovered_Fumbles"] = single_params[3]
            Def_Lib[Week][team_name]["Ints"] = single_params[4]
            Def_Lib[Week][team_name]["DefTD"] = single_params[5]
            Def_Lib[Week][team_name]["PA"] = single_params[6]
            Def_Lib[Week][team_name]["PaYd"] = single_params[7]
            Def_Lib[Week][team_name]["RuYd"] = single_params[8]
            Def_Lib[Week][team_name]["Safeties"] = single_params[9]
            Def_Lib[Week][team_name]["DefTD"] = single_params[10]
            Def_Lib[Week][team_name]["FanPts"] = single_params[11]
        else:
            Week += 1
    return Def_Lib
Example #13
0
def calculate_genfiles(source):
    with open(source, "r") as thrift:
        lines = thrift.readlines()
        namespaces = {}
        types = defaultdict(set)
        for line in lines:
            match = NAMESPACE_PARSER.match(line)
            if match:
                lang = match.group(1)
                namespace = match.group(2)
                namespaces[lang] = namespace
            else:
                match = TYPE_PARSER.match(line)
                if match:
                    type = match.group(1)
                    name = match.group(2)
                    types[type].add(name)

        genfiles = defaultdict(set)

        namespace = namespaces.get("py")
        if namespace:
            genfiles["py"].update(calculate_python_genfiles(namespace, types))

        namespace = namespaces.get("java")
        if namespace:
            genfiles["java"].update(calculate_java_genfiles(namespace, types))

        return genfiles
def ngram_unsmoothing_prob(i, ngram_letter_counter, n_1gram_letter_counter, all_letters_nr):
    ngram_prob_list = collections.defaultdict(int)
    prob_list = collections.defaultdict(int)

    if i == 1:
        for ngram in ngram_letter_counter:
            ngram_prob_list[ngram] = (ngram_letter_counter[ngram] + 0.0) / all_letters_nr

    else:
        for ngram in ngram_letter_counter:
            # print ngram
            # print ngram[0:i-1]
            list = ngram.split()
            n_1gram = list[0]
            if n_1gram_letter_counter[n_1gram] != 0:
                # print ngram_letter_counter[ngram]
                # print n_1gram_letter_counter[ngram[0:i-1]]
                ngram_prob_list[ngram] = (ngram_letter_counter[ngram] + 0.0) / n_1gram_letter_counter[n_1gram]
                # print ngram_prob_list[ngram]
            else:
                ngram_prob_list[ngram] = 0.0
            # print '~'*80

    for ngram, value in ngram_prob_list.iteritems():
        if value != 0:
            prob_list[ngram] = value
    return prob_list
Example #15
0
def process_results(entries, reverse_ranks):
    evalset_score_max = collections.defaultdict(lambda: 10)
    evalset_score_min = collections.defaultdict(lambda: 1000000000)
    for entry in entries:
        for result in entry["results"]:
            evalset = result["evalset"]
            if result["score"] >= 0:
                evalset_score_max[evalset] = max(evalset_score_max[evalset], result["score"])
                evalset_score_min[evalset] = min(evalset_score_min[evalset], result["score"])
    evalsets = sorted(evalset_score_max)
    for entry in entries:
        results_map = {}
        for result in entry["results"]:
            results_map[result["evalset"]] = result
        entry["results"] = [results_map.get(evalset) for evalset in evalsets]
        for result in entry["results"]:
            if result:
                if result["score"] >= 0:
                    result["score_ratio"] = 1.0 * result["score"] / evalset_score_max[result["evalset"]]
                else:
                    result["score_ratio"] = 1.0 if reverse_ranks else 0.0
                result["score_percent"] = int(result["score_ratio"] * 100)
                if reverse_ranks:
                    result["winner"] = False
                else:
                    result["winner"] = result["score"] >= evalset_score_max[result["evalset"]] * 0.9
        scores = [result["score_ratio"] for result in entry["results"] if result and result["score"] >= 0]
        entry["score_ratio"] = sum(scores) / len(scores) if scores else (1 if reverse_ranks else 0)
        entry["score_percent"] = int(100 * entry["score_ratio"])
    entries.sort(key=lambda entry: entry["score_ratio"], reverse=not reverse_ranks)
    return evalsets
Example #16
0
    def meta_compile_column_names(self):
        """Collect all of the column names from private school schemas and, along with the
        name map, produce a new name transformation spreadheet that can be used both for creating a schema
        and for updating the name transformations. """
        from collections import defaultdict, OrderedDict
        import csv

        b = self.library.dep("schools")

        fields = defaultdict(set)
        lengths = defaultdict(int)
        all_years = set()

        for table in b.schema.tables:
            if "private" not in table.name:
                continue

            _, _, year = table.name.split("_")
            year = int(year)
            all_years.add(year)

            for c in table.columns:
                fields[c.name].add(year)
                lengths[c.name] = c.size

        all_years = list(sorted(all_years))
        rows = []

        # Critically, field_names is in the order that the
        # fields should appear in the final schema, although it also
        # containes the names of fields that will be mapped to other names
        # and won't appear in the final schema.
        name_map, field_names = self.get_name_map()

        for i, field_name in enumerate(field_names):

            years = list(all_years)
            for j, year in enumerate(all_years):

                if year in fields[field_name]:
                    years[j] = "X"
                else:
                    years[j] = ""

            c = len(fields[field_name])

            tf = name_map[field_name]
            lengths[tf] = max(lengths[tf], lengths[field_name])

            rows.append([i, tf, field_name, lengths[tf], c] + years)

        # now make one more pass to ensure that all of the lengths are set.
        for i in range(len(rows)):
            rows[i][3] = max(lengths[tf], rows[i][3])

        with open(self.filesystem.path(self.metadata.build.revised_name_transforms), "wb") as f:
            writer = csv.writer(f)
            writer.writerow(["i", "to_name", "from_name", "size", "years"] + all_years)
            for row in rows:
                writer.writerow(row)
Example #17
0
def get_tag_sets(add_supersets=False, prefetch=True):
    tag_sets = defaultdict(set)
    tag_supersets = defaultdict(set)
    project_ids = []

    # Build up data structure that maps tag sets to
    # projects. These projects include projects with
    # tag supersets.
    #
    # The query retrieval can be made much faster
    # when the package django-batch-select is used:
    # for p in Project.objects.batch_select('tags'):
    #     tags = p.tags_all
    #     ...
    # However, this new dependency isn't justified if
    # only used here (in an admin tool).
    for p in Project.objects.all():
        project_ids.append(p.id)
        tags = p.tags.all()
        if len(tags) == 0:
            continue
        tag_sets[frozenset(tags)].add(p)

    if add_supersets:
        for a, b in combinations(tag_sets, 2):
            if a < b:
                tag_sets[a].update(tag_sets[b])
                tag_supersets[a].update(tag_sets[b])
            elif b < a:
                tag_sets[b].update(tag_sets[a])
                tag_supersets[b].update(tag_sets[a])

    return project_ids, tag_sets, tag_supersets
    def transform_funnel_data(self, funnel_data, table_view=False):
        # reduce all steps related to funnels grouped by date in one list
        funnel_step_count_by_day = [step for info in funnel_data["data"].values() for step in info["steps"]]
        # now aggregate the list of dictionaries
        funnel_steps = defaultdict(int)
        for steps in funnel_step_count_by_day:
            funnel_steps[steps["event"]] += steps["count"]

        if not table_view:
            data = defaultdict(list)
            for category, value in sorted(funnel_steps.iteritems(), key=lambda x: x[1], reverse=True):
                data["categories"].append(category)
                data["series"].append(value)
            return data
        else:  # transform data to table
            data = []
            row = OrderedDict()
            row["Funnel"] = "Overall"
            prev_step_val = None
            for category, value in sorted(funnel_steps.iteritems(), key=lambda x: x[1], reverse=True):
                if prev_step_val is None:
                    prev_step_val = value
                row[category] = "%d (%.2f%%)" % (value, (100 * float(value) / float(prev_step_val)))
                prev_step_val = value
            data.append(row)
            return data
Example #19
0
 def merge_tops(self, tops):
     """
     Cleanly merge the top files
     """
     top = collections.defaultdict(dict)
     orders = collections.defaultdict(dict)
     for ctops in tops.values():
         for ctop in ctops:
             for saltenv, targets in ctop.items():
                 if saltenv == "include":
                     continue
                 for tgt in targets:
                     matches = []
                     states = OrderedDict()
                     orders[saltenv][tgt] = 0
                     for comp in ctop[saltenv][tgt]:
                         if isinstance(comp, dict):
                             if "match" in comp:
                                 matches.append(comp)
                             if "order" in comp:
                                 order = comp["order"]
                                 if not isinstance(order, int):
                                     try:
                                         order = int(order)
                                     except ValueError:
                                         order = 0
                                 orders[saltenv][tgt] = order
                         if isinstance(comp, string_types):
                             states[comp] = True
                     top[saltenv][tgt] = matches
                     top[saltenv][tgt].extend(list(states.keys()))
     return self.sort_top_targets(top, orders)
Example #20
0
    def _source(self, bysock):
        conns = 0

        clients = defaultdict(lambda: 0)
        for sock in bysock:
            try:
                host, port = sock.getpeername()
            except socket.error:
                host, port = None, None

            clients[host] += 1
            conns += 1

        ordered = defaultdict(list)
        for host, number in clients.items():
            ordered[number].append(host)

        result = []
        result.append('<div style="padding: 10px 10px 10px 10px; font-weight:bold;">ExaProxy Statistics</div><br/>')
        result.append("<center>we have %d connection(s) from %d source(s)</center><br/>" % (conns, len(clients)))
        for number in reversed(sorted(ordered)):
            for host in ordered[number]:
                result.append('<span class="key">%s</span><span class="value">&nbsp; %s</span><br/>' % (host, number))

        return _listing % "\n".join(result)
def weight_bonds(words, idfs, mol):
    # Compute the word matches for bonds in the molecule
    bond_words = defaultdict(set)
    # Keep track of counts for use in TF-IDF later
    doc_word_counts = defaultdict(float)
    for i, w in enumerate(words):
        # mol_matches = mol.GetSubstructMatches(w,uniquify=False)
        mol_matches = mol.GetSubstructMatches(w, uniquify=False)
        if mol_matches:
            doc_word_counts[i] += len(mol_matches)
            for m in mol_matches:
                cmap = dict(enumerate(m))
                for b in w.GetBonds():
                    start = b.GetBeginAtomIdx()
                    end = b.GetEndAtomIdx()
                    bond_words[frozenset([cmap[start], cmap[end]])].add(i)

    # Compute the maximal words
    # words_to_use = set(maximal_words(words,doc_word_counts.keys()))
    words_to_use = doc_word_counts.keys()

    # Compute the TF-IDF scores for each word
    maxtf = float(max(doc_word_counts[t] for t in words_to_use))
    score = defaultdict(float, ((t, doc_word_counts[t] / maxtf * idfs[t]) for t in words_to_use))
    # Alternate, modified TF-IDF score
    # score = defaultdict(float,((t,log(1+doc_word_counts[t]/maxtf)*idfs[t])
    #                          for t in maxwords))

    # Get the combined TF-IDF scores for each bond
    bond_weights = dict((k, sum(score[t] for t in v)) for k, v in bond_words.items())
    # Return the bond values
    return bond_weights
def audit(osmfile):
    osm_file = open(osmfile, "r")

    street_types_file_out = "audit_street_types.json"
    street_types = defaultdict(set)

    cardinal_dirs_file_out = "audit_cardinal_dirs.json"
    cardinal_dirs = defaultdict(set)

    for event, elem in ET.iterparse(osm_file, events=("start",)):

        if elem.tag == "node" or elem.tag == "way":
            for tag in elem.iter("tag"):
                if is_street_name(tag):
                    # clean first, just like you'll do in data.py
                    street_name = tag.attrib["v"]
                    street_name = update_street_type(street_name)
                    street_name = update_cardinal_types(street_name)
                    audit_street_type(street_types, street_name)
                    audit_cardinal_dir(cardinal_dirs, street_name)

    osm_file.close()

    for s in street_types:
        street_types[s] = list(street_types[s])
    with codecs.open(street_types_file_out, "w") as fo:
        fo.write(json.dumps(street_types, indent=2) + "\n")

    for s in cardinal_dirs:
        cardinal_dirs[s] = list(cardinal_dirs[s])
    with codecs.open(cardinal_dirs_file_out, "w") as fo:
        fo.write(json.dumps(cardinal_dirs, indent=2) + "\n")

    return street_types
Example #23
0
    def __init__(self, *, conn_timeout=None, keepalive_timeout=sentinel, force_close=False, limit=20, loop=None):

        if force_close:
            if keepalive_timeout is not None and keepalive_timeout is not sentinel:
                raise ValueError("keepalive_timeout cannot " "be set if force_close is True")
        else:
            if keepalive_timeout is sentinel:
                keepalive_timeout = 30

        if loop is None:
            loop = asyncio.get_event_loop()

        self._closed = False
        if loop.get_debug():
            self._source_traceback = traceback.extract_stack(sys._getframe(1))

        self._conns = {}
        self._acquired = defaultdict(set)
        self._conn_timeout = conn_timeout
        self._keepalive_timeout = keepalive_timeout
        self._cleanup_handle = None
        self._force_close = force_close
        self._limit = limit
        self._waiters = defaultdict(list)

        self._loop = loop
        self._factory = functools.partial(aiohttp.StreamProtocol, loop=loop, disconnect_error=ServerDisconnectedError)

        self.cookies = http.cookies.SimpleCookie()
Example #24
0
def hierarchical(keys):
    """
    Iterates over dimension values in keys, taking two sets
    of dimension values at a time to determine whether two
    consecutive dimensions have a one-to-many relationship.
    If they do a mapping between the first and second dimension
    values is returned. Returns a list of n-1 mappings, between
    consecutive dimensions.
    """
    ndims = len(keys[0])
    if ndims <= 1:
        return True
    dim_vals = list(zip(*keys))
    combinations = (zip(*dim_vals[i : i + 2]) for i in range(ndims - 1))
    hierarchies = []
    for combination in combinations:
        hierarchy = True
        store1 = defaultdict(list)
        store2 = defaultdict(list)
        for v1, v2 in combination:
            if v2 not in store2[v1]:
                store2[v1].append(v2)
            previous = store1[v2]
            if previous and previous[0] != v1:
                hierarchy = False
                break
            if v1 not in store1[v2]:
                store1[v2].append(v1)
        hierarchies.append(store2 if hierarchy else {})
    return hierarchies
Example #25
0
 def flush(self):
     """ Garbage collect any previous pending messages """
     num_pending_messages = len(self.pendingreceive2conn_messages) + len(self.pendingsend2conn_messages)
     if num_pending_messages > 0:
         log.info("Flushing %d pending messages" % num_pending_messages)
     self.pendingreceive2conn_messages = defaultdict(list)
     self.pendingsend2conn_messages = defaultdict(list)
Example #26
0
def main():
    define("console", default=False, type=bool)
    define("mongo_host", default="localhost")

    parse_command_line()
    basicConfig(options=options)
    country_db = CountryDB(options.mongo_host)
    company_db = CompanyDB(options.mongo_host)
    stats_db = StatsDB(options.mongo_host)
    # companies = company_db.find({}, fields={'offices': 1}, count=None)
    ##for company in adv_enumerate(companies):
    ##    for office in company['offices']:
    ##        country_db.increment(office['country_code'])
    ##print company_db
    # _create_stats(stats_db, company_db)
    _create_csv(stats_db, company_db)
    return -1
    categories = defaultdict(int)
    year_month = defaultdict(int)
    year = defaultdict(int)
    for stats in adv_enumerate(stats_db.find()):
        categories[get_dotted(stats, "data.category_code")] += 1
        founded_at = get_dotted(stats, "data.founded_at")
        if not founded_at:
            continue
        # if founded_at.year < 1995:
        #    print stats
        year_month[(founded_at.year, founded_at.month)] += 1
        year[founded_at.year] += 1

    print sum(categories.values())
    print year
    print sorted(year_month.items())
Example #27
0
    def __init__(self):

        # Call the superclass constructor.
        super(AbstractNotifier, self).__init__()

        # Info message notification list for plugins
        # that receive all information types.
        # list(str)
        self._notification_info_all = []

        # Info message notification mapping for plugins.
        # dict(str -> str)
        self._notification_info_map = defaultdict(set)

        # Relationship notification mappings for plugins.
        # dict(str -> str)
        self._notification_rel_map_left = defaultdict(set)
        self._notification_rel_map_right = defaultdict(set)

        # Control message notification mapping for plugins.
        # list(Plugin)
        self._notification_msg_list = []

        # Map of plugin IDs to plugin instances.
        # dict(str -> Plugin)
        self._map_id_to_plugin = {}
Example #28
0
    def __init__(self):
        self.sponges = defaultdict(Block3DSpatialDict)
        self.springs = defaultdict(Block2DSpatialDict)

        self.pending = defaultdict(set)

        self.loop = LoopingCall(self.process)
 def __init__(self, view, off_path_hits=False, per_node=True, content_hits=False):
     """Constructor
     
     Parameters
     ----------
     view : NetworkView
         The NetworkView instance
     off_path_hits : bool, optional
         If *True* also records cache hits from caches not on located on the
         shortest path. This metric may be relevant only for some strategies
     content_hits : bool, optional
         If *True* also records cache hits per content instead of just
         globally
     """
     self.view = view
     self.off_path_hits = off_path_hits
     self.per_node = per_node
     self.cont_hits = content_hits
     self.sess_count = 0
     self.cache_hits = 0
     self.serv_hits = 0
     if off_path_hits:
         self.off_path_hit_count = 0
     if per_node:
         self.per_node_cache_hits = collections.defaultdict(int)
         self.per_node_server_hits = collections.defaultdict(int)
     if content_hits:
         self.curr_cont = None
         self.cont_cache_hits = collections.defaultdict(int)
         self.cont_serv_hits = collections.defaultdict(int)
Example #30
0
def genvertices(shp, pcount):
    """
    A generator that yields chunks of vertices

    Assuming that the data is spatially autocorrelated within the shapefile
    this should generate a fair number of neighbors and avoid sparse dicts.

    This would be something interesting to test: what function does SA have
    on the performance of the algorithm?

    """
    stepper = int(500)  # Maybe too big for MPI?
    numPoly = len(shp)

    vcount = 0

    vertices = collections.defaultdict(set)  # 2.7%
    while vcount < stepper and pcount < numPoly:
        newvertices = shp.get(pcount).vertices  # 33%
        vcount += len(newvertices)
        for v in newvertices[:-1]:  # 8%
            vertices[v].add(pcount)  # 19.2%
        pcount += 1
    yield pcount, vertices
    vcount = 0
    vertices = collections.defaultdict(set)  # 2.7%