def normDAGl2Test(G_test, power):
    kern = nx.get_edge_attributes(G_test, "kern_unnorm")
    tran = nx.get_edge_attributes(G_test, "tran")

    kern = OrderedDict(sorted(kern.items(), key=lambda t: t[0]))
    val = kern.values()
    key = kern.keys()

    tran = OrderedDict(sorted(tran.items(), key=lambda t: t[0]))
    tran = tran.values()

    val = np.asarray(val, dtype=float)
    tran = np.asarray(tran, dtype=float)
    tran = np.log(1 / tran)  # logarithm weighting
    tran[tran == np.inf] = 0
    tran[np.isnan(tran)] = 0

    if power == 2:
        tran = np.square(tran)

    if len(val.shape) == 2:
        # kern = val/tran[:, None]
        kern = val * tran[:, None]  # avoid numeric problems when using logarithm weighting
        kern = normalize(kern, norm="l2", axis=0)
    else:
        kern = val * tran
        kern = kern / np.linalg.norm(kern)

    kern = dict(zip(key, kern))
    nx.set_edge_attributes(G_test, "kern", kern)

    return G_test
Example #2
1
 def recursive_split(rect_key, rect_coords, category_idx, split_dir, gap):
     """
     given a key of the boxes and the data to analyze,
     split the key into several keys stratificated by the given
     category in the assigned direction
     """
     ticks = []
     category = categories[category_idx]
     chiave = rect_key
     divisione = OrderedDict()
     for tipo in category:
         divisione[tipo] = 0.0
         for k, v in counted.items():
             if k[len(rect_key) - 1] != tipo:
                 continue
             if not all(k[k1] == v1 for k1, v1 in enumerate(rect_key[1:])):
                 continue
             divisione[tipo] += v
     totali = 1.0 * sum(divisione.values())
     if totali:  # check for empty categories
         divisione = OrderedDict((k, v / totali) for k, v in divisione.items())
     else:
         divisione = OrderedDict((k, 0.0) for k, v in divisione.items())
     prop = divisione.values()
     div_keys = divisione.keys()
     new_rects = split_rect(*rect_coords, proportion=prop, direction=split_dir, gap=gap)
     divisi = OrderedDict((chiave + (k,), v) for k, v in zip(div_keys, new_rects))
     d = split_dir == "h"
     ticks = [(k, O[d] + 0.5 * [h, w][d]) for k, (O, h, w) in zip(div_keys, new_rects)]
     return divisi, zip(*ticks)
def get_field_info(model):
    """
    Given a model class, returns a `FieldInfo` instance containing metadata
    about the various field types on the model.
    """
    # Deal with the primary key.
    pk = model.id if not issubclass(model, mongoengine.EmbeddedDocument) else None

    # Deal with regular fields.
    fields = OrderedDict()

    for field_name in model._fields_ordered:
        fields[field_name] = model._fields[field_name]

    # Deal with forward relationships.
    # Pass forward relations since there is no relations on mongodb
    forward_relations = OrderedDict()

    # Deal with reverse relationships.
    # Pass reverse relations since there is no relations on mongodb
    reverse_relations = OrderedDict()

    # Shortcut that merges both regular fields and the pk,
    # for simplifying regular field lookup.
    fields_and_pk = OrderedDict()
    fields_and_pk["pk"] = pk
    fields_and_pk[getattr(pk, "name", "pk")] = pk
    fields_and_pk.update(fields)

    # Shortcut that merges both forward and reverse relationships

    relations = OrderedDict(list(forward_relations.items()) + list(reverse_relations.items()))

    return FieldInfo(pk, fields, forward_relations, reverse_relations, fields_and_pk, relations)
def main():
    directions_dir = Path("/HOME/huziy/skynet3_rech1/directions_for_ManitobaHydro")

    # Create the directory for the shapes
    shp_dir = directions_dir.joinpath("shp_direction_data_shortnames")
    if not shp_dir.is_dir():
        shp_dir.mkdir()

    grid_config_to_dirfile = OrderedDict(
        [
            (default_domains.bc_mh_044, directions_dir.joinpath("directions_mh_0.44deg.nc")),
            (default_domains.bc_mh_011, directions_dir.joinpath("directions_mh_0.11deg.nc")),
            (default_domains.bc_mh_022, directions_dir.joinpath("directions_mh_0.22deg.nc")),
        ]
    )

    for gc, dir_file in grid_config_to_dirfile.items():

        out_shp_filename = "{}.shp".format(dir_file.name[:-3])

        assert isinstance(gc, GridConfig)

        fields_to_add = OrderedDict()
        with Dataset(str(dir_file)) as ds:
            for vname, var in ds.variables.items():
                fields_to_add[vname] = var[:].view(MyNdArray)
                fields_to_add[vname].type_of_shp_field = "float"
                print("{} ranges from {} to {}".format(vname, fields_to_add[vname].min(), fields_to_add[vname].max()))

        # Export the cells to a shapefile
        fields_to_add = {field_name_to_shp_name[k]: v for k, v in fields_to_add.items()}

        gc.export_to_shape_fiona(shp_folder=shp_dir, shp_filename=out_shp_filename, shape_fields=fields_to_add)
class FioJobSection(object):
    def __init__(self, name):
        self.name = name
        self.vals = OrderedDict()

    def copy(self):
        return copy.deepcopy(self)

    def required_vars(self):
        for name, val in self.vals.items():
            if isinstance(val, Var):
                yield name, val

    def is_free(self):
        return len(list(self.required_vars())) == 0

    def __str__(self):
        res = "[{0}]\n".format(self.name)

        for name, val in self.vals.items():
            if name.startswith("_") or name == name.upper():
                continue
            if isinstance(val, Var):
                res += "{0}={{{1}}}\n".format(name, val.name)
            else:
                res += "{0}={1}\n".format(name, val)

        return res
Example #6
1
def _diff_trees(lhs, rhs, index, *path):
    def _fq(name):
        return "/".join(reversed((name,) + path))

    # Diff the trees (and keep deterministic order)
    rhs_tree_ids = OrderedDict((o.name, o.id) for o in rhs.tree_ids)
    for o in lhs.tree_ids:
        # remove so won't be picked up as added, below
        rhs_id = rhs_tree_ids.pop(o.name, None)
        if rhs_id == o.id:  # no change
            continue
        elif rhs_id is None:  # removed
            yield (_fq(o.name), o.id, None)
            rhs_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[])
        else:  # changed
            rhs_tree = index[rhs_id]
        for difference in _diff_trees(index[o.id], rhs_tree, index, o.name, *path):
            yield difference
    for name, id in rhs_tree_ids.items():  # added
        yield (_fq(name), None, id)
        lhs_tree = Object(_id=None, tree_ids=[], blob_ids=[], other_ids=[])
        for difference in _diff_trees(lhs_tree, index[id], index, name, *path):
            yield difference
    # Diff the blobs (and keep deterministic order)
    rhs_blob_ids = OrderedDict((o.name, o.id) for o in rhs.blob_ids)
    for o in lhs.blob_ids:
        rhs_id = rhs_blob_ids.pop(o.name, None)
        if rhs_id == o.id:
            continue  # no change
        elif rhs_id is None:
            yield (_fq(o.name), o.id, None)
        else:
            yield (_fq(o.name), o.id, rhs_id)
    for name, id in rhs_blob_ids.items():
        yield (_fq(name), None, id)
class InfoRow:
    def __init__(self):
        self.dict = OrderedDict()

    def __getitem__(self, item):
        if isinstance(item, int):
            return self.dict.items()[item][1]
        return self.dict.__getitem__(item.lower())

    def __contains__(self, item):
        return self.dict.__contains__(item.lower())

    def __setitem__(self, key, value):
        if isinstance(key, int):
            self.dict.items()[key][1] = value
        self.dict.__setitem__(key.lower(), value)

    def get(self, k, d=None):
        if isinstance(k, int):
            try:
                return self.dict.items()[k][1]
            except:
                return d
        return self.dict.get(k.lower(), d)

    def __str__(self, rowSeparator="\n", columnSeparator="\t"):
        return getStr(dict2Str(self.dict))

    def __unicode__(self):
        return str(self)

    def __repr__(self):
        return str(self)
Example #8
1
def htmlFromIndexTerms(entries):
    # entries: dict (preferably OrderedDict, if you want stability) of linkText=>{url, label, disambiguator}
    # label is used for the actual link (normally heading level), disambiguator is phrase to use when there are collisions

    from collections import OrderedDict

    entries = OrderedDict(sorted(entries.items(), key=lambda x: re.sub(r"[^a-z0-9]", "", x[0].lower())))

    topList = E.ul({"class": "index"})
    for text, items in entries.items():
        if len(items) == 1:
            item = items[0]
            li = appendChild(
                topList, E.li(E.a({"href": item["url"]}, text), E.span(", in ", item["label"]) if item["label"] else "")
            )
        else:
            li = appendChild(topList, E.li(text))
            ul = appendChild(li, E.ul())
            for item in items:
                appendChild(
                    ul,
                    E.li(
                        E.a({"href": item["url"]}, item["disambiguator"]),
                        E.span(", in ", item["label"]) if item["label"] else "",
                    ),
                )
    return topList
Example #9
1
File: io.py Project: JJMEI/mxnet
    def __init__(self, data, label=None, batch_size=1, shuffle=False, last_batch_handle="pad"):
        # pylint: disable=W0201

        super(NDArrayIter, self).__init__()

        self.data = _init_data(data, allow_empty=False, default_name="data")
        self.label = _init_data(label, allow_empty=True, default_name="softmax_label")

        # shuffle data
        if shuffle:
            idx = np.arange(self.data[0][1].shape[0])
            np.random.shuffle(idx)
            self.data = [(k, v[idx]) for k, v in self.data]
            self.label = [(k, v[idx]) for k, v in self.label]

        self.data_list = [x[1] for x in self.data] + [x[1] for x in self.label]
        self.num_source = len(self.data_list)

        # batching
        if last_batch_handle == "discard":
            new_n = self.data_list[0].shape[0] - self.data_list[0].shape[0] % batch_size
            data_dict = OrderedDict(self.data)
            label_dict = OrderedDict(self.label)
            for k, _ in self.data:
                data_dict[k] = data_dict[k][:new_n]
            for k, _ in self.label:
                label_dict[k] = label_dict[k][:new_n]
            self.data = data_dict.items()
            self.label = label_dict.items()
        self.num_data = self.data_list[0].shape[0]
        assert self.num_data >= batch_size, "batch_size need to be smaller than data size when not padding."
        self.cursor = -batch_size
        self.batch_size = batch_size
        self.last_batch_handle = last_batch_handle
Example #10
1
    def _classes_in_config_sample(self):
        """
        Yields only classes with own traits, and their subclasses.

        Thus, produced sample config-file will contain all classes
        on which a trait-value may be overridden:

        - either on the class owning the trait,
        - or on its subclasses, even if those subclasses do not define
          any traits themselves.
        """
        cls_to_config = OrderedDict(
            (cls, bool(cls.class_own_traits(config=True))) for cls in self._classes_inc_parents()
        )

        def is_any_parent_included(cls):
            return any(b in cls_to_config and cls_to_config[b] for b in cls.__bases__)

        ## Mark "empty" classes for inclusion if their parents own-traits,
        #  and loop until no more classes gets marked.
        #
        while True:
            to_incl_orig = cls_to_config.copy()
            cls_to_config = OrderedDict(
                (cls, inc_yes or is_any_parent_included(cls)) for cls, inc_yes in cls_to_config.items()
            )
            if cls_to_config == to_incl_orig:
                break
        for cl, inc_yes in cls_to_config.items():
            if inc_yes:
                yield cl
    def __init__(self, dictionary):
        self.ES_source = ""
        self.NSF = ""
        self.NSP = ""
        self.NSM = ""
        self.Crop = ""
        self.Pesticide = ""
        self.IUCN_Amphibians = ""
        self.IUCN_Birds = ""
        self.IUCN_Mammals = ""
        self.IUCN_Mammals_Marine = ""
        self.IUCN_Coral = ""
        self.IUCN_Reptiles = ""
        self.IUCN_Seagrasses = ""
        self.IUCN_SeaCucumbers = ""
        self.IUCN_Mangrove = ""
        self.IUCN_MarineFish = ""
        self.USFWS_p = ""
        self.USFWS_l = ""

        dictionary = OrderedDict(sorted(dictionary.items(), key=lambda t: t[0]))
        logger.info("===================")
        dictionary = OrderedDict(dictionary)
        for k, v in dictionary.items():
            setattr(self, k, v)
Example #12
0
    def print_search_packages(self, packages_props, reference, recipe_hash, packages_query):
        if not packages_props:
            if packages_query:
                warn_msg = "There are no packages for reference '%s' matching the query '%s'" % (
                    str(reference),
                    packages_query,
                )
            else:
                warn_msg = "There are no packages for pattern '%s'" % str(reference)
            self._out.info(warn_msg)
            return

        self._out.info("Existing packages for recipe %s:\n" % str(reference))
        # Each package
        for package_id, properties in sorted(packages_props.items()):
            self._print_colored_line("Package_ID", package_id, 1)
            for section in ("options", "settings", "full_requires"):
                attrs = properties.get(section, [])
                if attrs:
                    section_name = {"full_requires": "requires"}.get(section, section)
                    self._print_colored_line("[%s]" % section_name, indent=2)
                    if isinstance(attrs, dict):  # options, settings
                        attrs = OrderedDict(sorted(attrs.items()))
                        for key, value in attrs.items():
                            self._print_colored_line(key, value=value, indent=3)
                    elif isinstance(attrs, list):  # full requires
                        for key in sorted(attrs):
                            self._print_colored_line(key, indent=3)
            package_recipe_hash = properties.get("recipe_hash", None)
            # Always compare outdated with local recipe, simplification, if a remote check is needed install recipe first
            if recipe_hash:
                self._print_colored_line("outdated from recipe: %s" % (recipe_hash != package_recipe_hash), indent=2)
            self._out.writeln("")
Example #13
0
class StatsMessage(Message):
    """
    Contains detailed information about one CS.

    Format is <host_id>[body]
    """

    msg_type = 0x57

    def __init__(self, host_id, body):
        self.host_id = host_id
        self.parse_body(body)

    def parse_body(self, body):
        self.data = OrderedDict()
        for l in body.split(b"\n"):
            vals = l.split(b":")
            if len(vals) == 2:
                self.data[vals[0].decode()] = vals[1].decode()

    def pack(self):
        body = b"\n".join(b":".join(map(lambda s: s.encode(), i)) for i in self.data.items())
        return struct.pack("!LL", self.host_id, len(body)) + body + b"\x00"

    @classmethod
    def unpack(cls, string):
        (host_id,) = struct.unpack("!L", string[:4])

        body, remainder = cls.unpack_string(string[4:])
        assert not remainder
        return StatsMessage(host_id, body)

    def __str__(self):
        body = "\n".join(": ".join(i) for i in self.data.items())
        return "[STATS] host = {0}, \n{1}".format(self.host_id, body)
Example #14
0
    def writeParams(self, runPath, pDict, runNumStr, seedStr):
        paramFilename = os.path.join(runPath, self.config.parametersFilename)

        runPDict = OrderedDict()
        runPDict.update(self.config.allConstants)
        runPDict[self.config.runNumberParameterName] = runNumStr
        runPDict[self.config.randomSeedParameterName] = seedStr
        runPDict.update(pDict)

        if self.config.parametersFormat == "json":
            runPDictHierarchical = makeHierarchicalDict(runPDict)

            dumpJson(runPDictHierarchical, paramFilename)
        elif self.config.parametersFormat == "csv":
            csvFile = open(paramFilename, "w")
            csvWriter = csv.writer(csvFile)
            for key, value in runPDict.items():
                csvWriter.writerow((key, value))
            csvFile.close()
        elif self.config.parametersFormat == "txt":
            csvFile = open(paramFilename, "w")
            csvWriter = csv.writer(csvFile, delimiter="\t")
            for key, value in runPDict.items():
                csvWriter.writerow((key, value))
            csvFile.close()
def test():
    from collections import OrderedDict as StdlibOrderedDict

    ordered_dict = OrderedDict(((1, "a"), (2, "b"), (3, "c")))
    stdlib_ordered_dict = StdlibOrderedDict(((1, "a"), (2, "b"), (3, "c")))

    assert ordered_dict == stdlib_ordered_dict
    assert stdlib_ordered_dict == ordered_dict
    assert ordered_dict.items() == stdlib_ordered_dict.items()
    assert ordered_dict.keys() == stdlib_ordered_dict.keys()
    assert ordered_dict.values() == stdlib_ordered_dict.values()

    ordered_dict.move_to_end(1)

    assert ordered_dict != stdlib_ordered_dict
    # assert stdlib_ordered_dict != ordered_dict
    assert ordered_dict.items() != stdlib_ordered_dict.items()
    assert ordered_dict.keys() != stdlib_ordered_dict.keys()
    assert ordered_dict.values() != stdlib_ordered_dict.values()

    del stdlib_ordered_dict[1]
    stdlib_ordered_dict[1] = "a"

    assert ordered_dict == stdlib_ordered_dict
    assert stdlib_ordered_dict == ordered_dict
    assert ordered_dict.items() == stdlib_ordered_dict.items()
    assert ordered_dict.keys() == stdlib_ordered_dict.keys()
    assert ordered_dict.values() == stdlib_ordered_dict.values()

    assert ordered_dict == OrderedDict(stdlib_ordered_dict) == stdlib_ordered_dict
    assert ordered_dict == StdlibOrderedDict(ordered_dict) == stdlib_ordered_dict
def processImage():
    # url = request.args.get('url', '')
    url = request.form["url"]
    path = id_generator()
    path = "data/" + path
    downloadImages([url], path)
    testImages = gl.image_analysis.load_images(path, random_order=False, with_path=False)
    images_resized_test = gl.SFrame()
    images_resized_test["image"] = gl.image_analysis.resize(testImages["image"], 256, 256, 3)
    images_resized_test["extracted_features"] = pretrained_model.extract_features(images_resized_test)
    pred1 = cloth_classifier.predict_topk(images_resized_test, k=3)
    pred2 = color_classifier.predict_topk(images_resized_test, k=3)

    cloth_type = {}
    for row in pred1.to_numpy():
        cloth_type[row[1]] = row[2]
    cloth_type = OrderedDict(sorted(cloth_type.items(), key=lambda x: x[1], reverse=True))

    color = {}
    for row in pred2.to_numpy():
        color[row[1]] = row[2]
    color = OrderedDict(sorted(color.items(), key=lambda x: x[1], reverse=True))
    # print color

    return json.dumps([cloth_type, color])
def makeAllCompare(result1, result2):
    """A short to make all compare bar diagramms, using the module makePicture.py
    
    
    :param result1: The first result data that stored locally after computation.py processing.
    :type result: Dict.
    :param result2: The second result data that stored locally after computation.py processing.
    :type result: Dict.
    :returns: None
    """
    # first sort this input dict
    from collections import OrderedDict

    result1 = OrderedDict(sorted(result1.items(), key=lambda t: t[0]))
    result2 = OrderedDict(sorted(result2.items(), key=lambda t: t[0]))
    # get iterator for this two dict
    i1 = iter(result1)
    i2 = iter(result2)
    try:
        while True:
            k1 = next(i1)
            dict1 = result1[k1]
            k2 = next(i2)
            dict2 = result2[k2]
            if len(dict1.keys()) < 5 or len(dict2.keys()) < 5:
                continue
            makeCompareBar(dict1, dict2, k1, k2)
    except StopIteration:
        pass
    finally:
        del i1, i2
def generate_docs():
    """The main function."""
    tree = read_xml()

    # Create a simple list to capture all of the STIGs
    stig_ids = []

    # Create defaultdicts to hold information to build our table of
    # contents files for sphinx.
    all_deployer_notes = defaultdict(list)
    severity = defaultdict(list)
    tag = defaultdict(list)
    status = defaultdict(list)

    # Loop through the groups and extract rules
    group_elements = filter_xpath(tree, "/x:Benchmark/x:Group")
    for group_element in group_elements:
        rule_element = filter_find(group_element, "x:Rule")

        # Build a dictionary with all of our rule data.
        rule = {
            "id": group_element.attrib["id"],
            "title": filter_find(rule_element, "x:title").text,
            "severity": rule_element.attrib["severity"],
            "fix": filter_find(rule_element, "x:fixtext").text,
            "check": filter_find(rule_element, "x:check/x:check-content").text,
            "ident": [x.text for x in filter_find(rule_element, "x:ident")],
        }

        # The description has badly formed XML in it, so we need to hack it up
        # and turn those tags into a dictionary.
        description = filter_find(rule_element, "x:description").text
        parser = etree.XMLParser(recover=True)
        temp = etree.fromstring("<root>{0}</root>".format(description), parser)
        rule["description"] = {x.tag: x.text for x in temp.iter()}

        # Get the deployer notes from YAML
        deployer_notes = get_deployer_notes(rule["id"])
        rule["deployer_notes"] = deployer_notes

        all_deployer_notes[rule["id"]] = rule
        stig_ids.append(rule["id"])
        severity[rule["severity"]].append(rule["id"])
        status[deployer_notes["status"]].append(rule["id"])
        tag[deployer_notes["tag"]].append(rule["id"])

    keyorder = ["high", "medium", "low"]
    severity = OrderedDict(sorted(severity.items(), key=lambda x: keyorder.index(x[0])))
    status = OrderedDict(sorted(status.items(), key=lambda x: x[0]))
    tag = OrderedDict(sorted(tag.items(), key=lambda x: x[0]))

    all_toc = render_all(stig_ids, all_deployer_notes)
    severity_toc = render_toc("severity", severity, all_deployer_notes)
    status_toc = render_toc("implementation status", status, all_deployer_notes)
    tag_toc = render_toc("tag", tag, all_deployer_notes)

    write_file("rhel7/auto_controls-all.rst", all_toc)
    write_file("rhel7/auto_controls-by-severity.rst", severity_toc)
    write_file("rhel7/auto_controls-by-status.rst", status_toc)
    write_file("rhel7/auto_controls-by-tag.rst", tag_toc)
Example #19
0
    def get_request(self, post_data=""):
        log.debug(["POST DATA", post_data])
        ## parse path..
        p = urlparse.urlparse(self.path)
        q_get = OrderedDict(urlparse.parse_qs(p.query))
        ## flatten..
        for k, v in q_get.items():
            if len(v) == 1:
                q_get[k] = v[0]

        ## totally does not currently handle any kind of
        ## multipart posting
        q_post = OrderedDict(urlparse.parse_qs(post_data))
        ## flatten..
        for k, v in q_post.items():
            if len(v) == 1:
                q_post[k] = v[0]

        q_request = OrderedDict(q_get.items() + q_post.items())

        return http.Request(
            client_address=self.client_address,
            server=self.server,
            method=self.command,
            headers=dict(self.headers),
            path=p.path,
            ## OrderedDict..
            GET=q_get,
            POST=q_post,
            REQUEST=q_request,
        )
Example #20
0
 def __repr__(self):
     repr_string = "Sequence name: {}\n".format(self.seq_name)
     clusters = OrderedDict(sorted(self.clusters.items()))
     cluster_nums = "\t".join([str(clust.clusters) for _, clust in clusters.items()])
     cluster_ids = "\t".join([str(dist) for dist, _ in clusters.items()])
     repr_string += "Clustering distance:\t{}\nTotal clusters:\t\t{}\n".format(cluster_ids, cluster_nums)
     return repr_string
Example #21
0
File: io.py Project: JJMEI/mxnet
def _init_data(data, allow_empty, default_name):
    """Convert data into canonical form."""
    assert (data is not None) or allow_empty
    if data is None:
        data = []

    if isinstance(data, (np.ndarray, NDArray)):
        data = [data]
    if isinstance(data, list):
        if not allow_empty:
            assert len(data) > 0
        if len(data) == 1:
            data = OrderedDict([(default_name, data[0])])
        else:
            data = OrderedDict([("_%d_%s" % (i, default_name), d) for i, d in enumerate(data)])
    if not isinstance(data, dict):
        raise TypeError("Input must be NDArray, numpy.ndarray, " + "a list of them or dict with them as values")
    for k, v in data.items():
        if isinstance(v, NDArray):
            data[k] = v.asnumpy()
    for k, v in data.items():
        if not isinstance(v, np.ndarray):
            raise TypeError(("Invalid type '%s' for %s, " % (type(v), k)) + "should be NDArray or numpy.ndarray")

    return list(data.items())
Example #22
0
def get_new_fail_msg(latest_status, latest_suites, prev_name, prev_status, prev_suites):
    """Returns 'new failures' message for email body, or empty string."""

    if not prev_status or not prev_suites:
        return ""

    latest_crashed = not latest_status.startswith("OK\n")
    prev_crashed = not prev_status.startswith("OK\n")
    if latest_crashed or prev_crashed:
        return ""

    prev_failures = {s.name: {t.name for t in s.testcases if t.err or t.fail} for s in prev_suites}
    latest_failures = OrderedDict(
        (s.name, OrderedDict((t.name, True) for t in s.testcases if t.err or t.fail)) for s in latest_suites
    )

    new_failures = OrderedDict()
    for s, latest_tests in latest_failures.items():
        new = latest_tests
        if s in prev_failures:
            for x in prev_failures[s]:
                new.pop(x, None)
        if new:
            new_failures[s] = new

    if not new_failures:
        return ""

    msg = "\n"
    msg += "*New failures* since " + prev_name + ":\n"
    for s, tests in new_failures.items():
        msg += "\n" + s + " (" + str(len(tests)) + "):\n"
        for t in tests:
            msg += t + "\n"
    return msg + "\n\n"
def hook_preconvert_03_by():
    global by_year, by_version, by_manufacturer, by_submitter, by_category
    by_year = OrderedDict(sorted(by_year.items()))
    by_version = OrderedDict(sorted(by_version.items()))
    by_manufacturer = OrderedDict(sorted(by_manufacturer.items()))
    by_submitter = OrderedDict(sorted(by_submitter.items()))
    by_category = OrderedDict(sorted(by_category.items()))
def IsMessageSpam(tokenProbabilities, tokens):
    # P(W1|S)*P(W2|S)*P(W3|S)*...
    priorProduct = 1
    # P(W1|S)*P(W2|S)*P(W3|S)*... + (1-P(W1|S))*(1-P(W2|S))*(1-P(W3|S))*...
    complementaryProduct = 1

    if len(tokens) >= 15:
        relevantTokens = {}
        for token in tokens:
            distinction = abs(0.5 - round(tokenProbabilities.get(token, 0.4), 7))
            if distinction in relevantTokens:
                relevantTokens[distinction].append(token)
            else:
                relevantTokens[distinction] = [token]

        relevantTokens = OrderedDict(sorted(relevantTokens.items(), reverse=True))
        tokens = []
        wordCount = 0
        for prob, words in relevantTokens.items():
            for word in words:
                if wordCount < 15 and prob > 0.1:
                    wordCount += 1
                    tokens.append(word)

    # Compute prior(evidence) product
    for token in tokens:
        priorProduct *= tokenProbabilities.get(token, 0.4)

    # Compute complementary product
    for token in tokens:
        complementaryProduct *= 1 - tokenProbabilities.get(token, 0.4)

    return round(priorProduct / (priorProduct + complementaryProduct), 2)
Example #25
0
    def data_to_docs(self):
        """ Turns data into a usable format for the TfidfVectorizer 
		
		INPUT: MongoDB data with 1 Mongo Document per tweet
		
		OUTPUT: Updates self.dict with OrderedDict. OrderedDict has 1 
		key per person, and the text is 1 long string with all of the tweets.
		"""

        client = MongoClient()
        processed_data = client["processed_data"]
        count_vect = processed_data["count_vect"]

        docs = count_vect.find({})
        doc_dict = OrderedDict()

        for doc in docs:
            tweet = doc.get("text").encode("utf8", "ignore")
            user = doc.get("user").get("screen_name")
            if user in doc_dict:
                doc_dict[user] += " " + tweet
            else:
                doc_dict[user] = tweet

        print doc_dict.items()[1]

        self.dict = doc_dict
        return self.dict
def get_initial_cluster_centroids(i_mat, k, rows, cols):
    # compute overall mean of the sample
    om_vec = np.zeros(cols)
    for r in range(0, rows):
        om_vec = np.add(om_vec, i_mat[r])
    om_vec = np.divide(om_vec, rows)

    # compute euclidian distance of each sample from overall mean
    uc_dict = {}
    for r in range(0, rows):
        dist = sci_dist.euclidean(i_mat[r], om_vec)
        uc_dict[r] = dist

    # sort samples based on euclidian distance of each sample from overall mean
    uc_dict = OrderedDict(sorted(uc_dict.items(), key=lambda x: x[1]))

    # get temporary sorted samples based on their euclidian distance of each
    # sample from overall mean
    t_i_mat = np.zeros(shape=(rows, cols))
    r = 0
    for key, val in uc_dict.items():
        t_i_mat[r] = i_mat[key]
        r += 1

    # get initial centroids
    centroids = {}
    for i in range(1, k + 1):
        n_arr = np.zeros(cols)
        c = 1 + (i - 1) * int(rows / k)
        n_arr = t_i_mat[c - 1]
        centroids[i - 1] = n_arr

    return centroids
Example #27
0
def walk_and_clean(data):
    """
    Recursively walks list of dicts (which may themselves embed lists and dicts),
    transforming namedtuples to OrderedDicts and
    using ``clean_key_name(k)`` to make keys into SQL-safe column names

    >>> data = [{'a': 1}, [{'B': 2}, {'B': 3}], {'F': {'G': 4}}]
    >>> pprint(walk_and_clean(data))
        [OrderedDict([('a', 1)]),
         [OrderedDict([('b', 2)]), OrderedDict([('b', 3)])],
          OrderedDict([('f', OrderedDict([('g', 4)]))])]
    """
    # transform namedtuples to OrderedDicts
    if hasattr(data, "_fields"):
        data = OrderedDict((k, v) for (k, v) in zip(data._fields, data))
    # Recursively clean up child dicts and lists
    if hasattr(data, "items") and hasattr(data, "__setitem__"):
        for (key, val) in data.items():
            data[key] = walk_and_clean(val)
    elif isinstance(data, list) or isinstance(data, tuple) or hasattr(data, "__next__") or hasattr(data, "next"):
        data = [walk_and_clean(d) for d in data]

    # Clean up any keys in this dict itself
    if hasattr(data, "items"):
        original_keys = data.keys()
        tup = ((clean_key_name(k), v) for (k, v) in data.items())
        data = OrderedDict(tup)
        if len(data) < len(original_keys):
            raise KeyError("Cleaning up %s created duplicates" % original_keys)
    return data
Example #28
0
def humans_per_hour(game, **kwargs):
    data = []
    end_date = min(timezone.now(), game.end_date)
    end_td = end_date - game.start_date
    end_hour = end_td.days * 24 + round(end_td.seconds / 3600, 0)
    for dorm, dormName in DORMS:
        sh = game.get_active_players().filter(dorm=dorm).count()  # starting humans in this dorm
        d = OrderedDict([(0, sh)])
        kills = Kill.objects.exclude(parent=None).filter(victim__game=game, victim__dorm=dorm).order_by("date")
        for index, kill in enumerate(kills, 1):
            kd = max(kill.date, game.start_date) - game.start_date
            hours = kd.days * 24 + round(kd.seconds / 3600, 1)
            d[min(hours, end_hour)] = sh - index  # overwrite
        if end_hour not in d:
            d[end_hour] = d[d.keys()[-1]]
        data.append({"name": dormName, "data": d.items()})
        # add dataset for all dorms
    sh = game.get_active_players().count() - Kill.objects.filter(parent=None, killer__game=game).count()  # subtract LZs
    d = OrderedDict([(0, sh)])
    kills = Kill.objects.exclude(parent=None).filter(victim__game=game).order_by("date")
    for index, kill in enumerate(kills, 1):
        kd = max(kill.date, game.start_date) - game.start_date
        hours = kd.days * 24 + round(kd.seconds / 3600, 1)
        d[min(hours, end_hour)] = sh - index  # overwrite
    if end_hour not in d:
        d[end_hour] = d[d.keys()[-1]]
    data.append({"name": "ALL", "data": d.items()})
    return data
Example #29
0
    def format(self, data, keys=None, group_by=None, domain=None):
        rows_dict = OrderedDict()
        tmp_data = OrderedDict()
        sorted_data = []
        value_chains = get_domain_configuration(domain).by_type_hierarchy
        for key, row in data.items():
            to_list = list(key)

            def find_name(list, deep):
                for element in list:
                    if deep == len(key) - 3 and key[deep + 1] == element.val:
                        return element.text
                    elif key[deep + 1] == element.val:
                        return find_name(element.next, deep + 1)

            name = find_name(value_chains, 0)
            to_list[2] = name
            tmp_data.update({tuple(to_list): row})
        if tmp_data:
            sorted_data = sorted(tmp_data.items(), key=lambda x: (x[0][0], x[0][2]))

        for row in sorted_data:
            formatted_row = self._format.format_row(row[1])
            if not rows_dict.has_key(formatted_row[0]):
                rows_dict[formatted_row[0]] = []
            rows_dict[formatted_row[0]].append(formatted_row[1])

        for key, row in rows_dict.items():
            total_column = self.calculate_total_column(row)
            res = [key, total_column]
            res.extend(row)
            yield res
def create_jobs():
    g1_components = range(1)
    g2_components = range(2)
    g3_components = range(2)
    component_groups = OrderedDict((("g1", g1_components), ("g2", g2_components), ("g3", g3_components)))

    api = mock_api.api(__file__)

    def job(name, expect_order, params=None):
        api.job(name, exec_time=0.5, max_fails=0, expect_invocations=1, expect_order=expect_order, params=params)

    api.flow_job()
    job("prepare", 1)
    for gname, group in component_groups.items():
        for component in group:
            job("deploy_component_" + gname + "_" + str(component), 2)
    job("report_deploy", 3)
    job("prepare_tests", 3)
    job("test_ui", 4)
    job("test_x", 4)
    for gname, group in component_groups.items():
        for component in group:
            job("test_component_" + gname + "_" + str(component), 5)
    job("report", 6, params=(("s1", "tst_regression", "desc"), ("c1", ("complete", "partial"), "desc")))
    job("promote", 7)
    return api