Example #1
1
def generate_log_stats(logs):
    response_codes = []
    data_size = 0
    host_regex = re.compile("https?://([^/]+)/.*$")
    all_hosts_data = {}
    for log in logs:
        with open(log, "rb") as l:
            for line in l:
                # Annotations can contain whitespace so limit the split.
                fields = line.split(None, 15)
                match = host_regex.match(fields[3])
                if match is not None:
                    host = match.group(1)
                    try:
                        host_data = all_hosts_data[host]
                    except KeyError:
                        all_hosts_data[host] = {"data_size": 0, "response_codes": []}
                        host_data = all_hosts_data[host]
                    host_data["response_codes"].append(fields[1])
                    if fields[2] != "-":
                        host_data["data_size"] += int(fields[2])
                    if "serverMaxSuccessKb" in fields[-1]:
                        host_data["reached_cap"] = True
    all_hosts_data = OrderedDict(sorted(all_hosts_data.iteritems(), key=operator.itemgetter(1), reverse=True))
    for host, data in all_hosts_data.iteritems():
        data["response_codes"] = Counter(data["response_codes"])
        data["data_size"] = human_readable(data["data_size"])
    return all_hosts_data
Example #2
1
 def run_cloners(cls, old_event, new_event):
     selected = {x[7:] for x in request.values.getlist("cloners") if x.startswith("cloner_")}
     all_cloners = OrderedDict((name, cloner_cls(old_event)) for name, cloner_cls in get_event_cloners().iteritems())
     if any(cloner.is_internal for name, cloner in all_cloners.iteritems() if name in selected):
         raise Exception("An internal cloner was selected")
     # enable internal cloners that are enabled by default or required by another cloner
     selected |= {
         c.name
         for c in all_cloners.itervalues()
         if c.is_internal and (c.is_default or c.required_by_deep & selected)
     }
     # enable unavailable cloners that may be pulled in as a dependency nonetheless
     extra = {
         c.name
         for c in all_cloners.itervalues()
         if not c.is_available and c.always_available_dep and c.required_by_deep & selected
     }
     selected |= extra
     active_cloners = OrderedDict((name, cloner) for name, cloner in all_cloners.iteritems() if name in selected)
     if not all(
         (c.is_internal or c.is_visible) and c.is_available
         for c in active_cloners.itervalues()
         if c.name not in extra
     ):
         raise Exception("An invisible/unavailable cloner was selected")
     for name, cloner in active_cloners.iteritems():
         if not (selected >= cloner.requires_deep):
             raise Exception("Cloner {} requires {}".format(name, ", ".join(cloner.requires_deep - selected)))
     shared_data = {}
     cloner_names = set(active_cloners)
     for name, cloner in active_cloners.iteritems():
         shared_data[name] = cloner.run(new_event, cloner_names, cloner._prepare_shared_data(shared_data))
Example #3
1
def getStatistics(keyfile, resfile, n=10):
    confusion = getConfusion(keyfile, resfile)
    correct = defaultdict(float)
    error = defaultdict(float)
    correctDist = defaultdict(float)
    errorDist = defaultdict(float)
    # for tag in alltags:
    for (true, pred), value in confusion.iteritems():
        if pred == true:
            correct[true] += value
        else:
            error[true] += value

    for tag, value in correct.iteritems():
        correctDist[tag] = correct[tag] / (correct[tag] + error[tag])

    for tag, value in error.iteritems():
        errorDist[tag] = error[tag] / (correct[tag] + error[tag])

    best = OrderedDict(sorted(correctDist.items(), key=lambda t: t[1], reverse=True)[:n])
    worst = OrderedDict(sorted(errorDist.items(), key=lambda t: t[1], reverse=True)[:n])

    print "----- Best prediction -----"
    for key, value in best.iteritems():
        print key, value, correct[key]

    print
    print "---- Worst Prediction ----"
    for key, value in worst.iteritems():
        print key, value, error[key]
    return best, worst
Example #4
1
def createEnumerated(element):
    valuemap = OrderedDict()
    name = "UA_" + element.get("Name")
    enum_types.append(name)
    print("\n/*** " + name + " ***/", end="\n", file=fh)
    for child in element:
        if child.tag == "{http://opcfoundation.org/BinarySchema/}Documentation":
            print("/* " + child.text + " */", end="\n", file=fh)
        if child.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedValue":
            valuemap[name + "_" + child.get("Name")] = child.get("Value")
    valuemap = OrderedDict(sorted(valuemap.iteritems(), key=lambda (k, v): int(v)))
    print("typedef UA_UInt32 " + name + ";", end="\n", file=fh)
    print(
        "enum "
        + name
        + "_enum { \n\t"
        + ",\n\t".join(map(lambda (key, value): key.upper() + " = " + value, valuemap.iteritems()))
        + "\n};",
        end="\n",
        file=fh,
    )
    print("UA_TYPE_METHOD_PROTOTYPES (" + name + ")", end="\n", file=fh)
    print("UA_TYPE_METHOD_CALCSIZE_AS(" + name + ", UA_UInt32)", end="\n", file=fc)
    print("UA_TYPE_METHOD_ENCODEBINARY_AS(" + name + ", UA_UInt32)", end="\n", file=fc)
    print("UA_TYPE_METHOD_DECODEBINARY_AS(" + name + ", UA_UInt32)", end="\n", file=fc)
    print("UA_TYPE_METHOD_DELETE_AS(" + name + ", UA_UInt32)", end="\n", file=fc)
    print("UA_TYPE_METHOD_DELETEMEMBERS_AS(" + name + ", UA_UInt32)", end="\n", file=fc)
    print("UA_TYPE_METHOD_INIT_AS(" + name + ", UA_UInt32)", end="\n", file=fc)
    print("UA_TYPE_METHOD_NEW_DEFAULT(" + name + ")\n", end="\n", file=fc)

    return
Example #5
1
class FreqDict(object):
    def __init__(self, terms, reverse=False):
        assert type(terms) in [list, set, np.ndarray], "input must be list, set or ndarray."
        fd = defaultdict(int)
        for term in terms:
            fd[term] += 1
        self.__fd = OrderedDict()
        for k, v in sorted(fd.iteritems(), key=lambda d: d[1], reverse=reverse):
            self.__fd[k] = v

    def __getitem__(self, item):
        return self.__fd[item]

    def __iter__(self):
        return iter(self.__fd.iteritems())

    def keys(self):
        return self.__fd.keys()

    def values(self):
        return self.__fd.values()

    def print_freq_dict(self):
        for k, v in self.__fd.iteritems():
            print k, v

    def plot_pie(self):
        plt.pie(x=self.__fd.values(), labels=self.__fd.keys(), autopct="%1.1f%%", shadow=True)
        plt.show()
Example #6
1
def processQuery(str):
    qtf = defaultdict(float)
    qtfidf = defaultdict(float)
    qwords = re.findall("\w+", str.lower(), re.UNICODE)
    sum = 0
    for word in qwords:
        qtf[word] = qtf[word] + 1
    for k, v in qtf.iteritems():
        qtf[k] = 1 + math.log(qtf[k], 2)
        qtfidf[k] = qtf[k] * idf[k]
        sum += qtfidf[k] * qtfidf[k]
        if qtfidf[k] == 0:
            print "no match"
            return
    norm = math.sqrt(sum)
    for i, j in qtfidf.iteritems():
        qtfidf[i] = qtfidf[i] / norm
    pgrank = buildRank(qtfidf)
    newdict = OrderedDict(sorted(pgrank.iteritems(), key=lambda t: t[1], reverse=True)[:50])
    topdict = OrderedDict()
    for key, val in newdict.iteritems():
        topdict[key] = val
    combdict = defaultdict(list)
    for key, val in topdict.iteritems():
        if topdict[key] != 0:
            score = tweetuserscore[tweetid[key]]
            combdict[key].append(val)
            combdict[key].append(score)
    finaldict = OrderedDict(sorted(combdict.iteritems(), key=lambda t: (t[1][0], t[1][1]), reverse=True)[:50])
    rank = 1
    for key, val in finaldict.iteritems():
        print "rank: ", rank, " tweetid: ", tweetid[key], " tweet: ", tweets[key].encode("utf-8")
        rank += 1
    return
Example #7
1
    def get_with_data(*args, **kwargs):
        filters = kwargs.pop("filters", None)
        limit = kwargs.pop("limit", None)
        offset = kwargs.pop("offset", 0)
        order = kwargs.pop("order", Reservation.start_dt)
        limit_per_room = kwargs.pop("limit_per_room", False)
        if kwargs:
            raise ValueError("Unexpected kwargs: {}".format(kwargs))

        query = Reservation.query.options(joinedload(Reservation.room))
        if filters:
            query = query.filter(*filters)
        if limit_per_room and (limit or offset):
            query = limit_groups(query, Reservation, Reservation.room_id, order, limit, offset)

        query = query.order_by(order, Reservation.created_dt)

        if not limit_per_room:
            if limit:
                query = query.limit(limit)
            if offset:
                query = query.offset(offset)

        result = OrderedDict((r.id, {"reservation": r}) for r in query)

        if "vc_equipment" in args:
            vc_id_subquery = (
                db.session.query(EquipmentType.id)
                .correlate(Reservation)
                .filter_by(name="Video conference")
                .join(RoomEquipmentAssociation)
                .filter(RoomEquipmentAssociation.c.room_id == Reservation.room_id)
                .as_scalar()
            )

            # noinspection PyTypeChecker
            vc_equipment_data = dict(
                db.session.query(Reservation.id, static_array.array_agg(EquipmentType.name))
                .join(ReservationEquipmentAssociation, EquipmentType)
                .filter(Reservation.id.in_(result.iterkeys()))
                .filter(EquipmentType.parent_id == vc_id_subquery)
                .group_by(Reservation.id)
            )

            for id_, data in result.iteritems():
                data["vc_equipment"] = vc_equipment_data.get(id_, ())

        if "occurrences" in args:
            occurrence_data = OrderedMultiDict(
                db.session.query(ReservationOccurrence.reservation_id, ReservationOccurrence)
                .filter(ReservationOccurrence.reservation_id.in_(result.iterkeys()))
                .order_by(ReservationOccurrence.start_dt)
            )
            for id_, data in result.iteritems():
                data["occurrences"] = occurrence_data.getlist(id_)

        return result.values()
Example #8
0
class AttributeList(object):
    def __init__(self):
        self._od = OrderedDict()

    def append(self, x):
        assert len(x) == 2
        tag = x[0]
        value = x[1]
        assert isinstance(tag, unicode)
        if tag in self._od:
            raise ValueError(b"Attributes must have unique tags")
        if not isinstance(value, Presentable):
            raise ValueError(b"Unrecognized attribute value type")
        self._od[tag] = value

    def as_table(self):
        val = "<table>"
        for k, v in self._od.iteritems():
            cell1 = html.escape(ugettext(k))
            cell2 = present(v)
            val += "<tr><th>{0}</th><td>{1}</td></tr>".format(cell1, cell2)
        val += "</table>"
        return val

    def find(self, tag, default=None):
        return self._od.get(tag, default)

    def __iter__(self):
        return self._od.iteritems()

    def __len__(self):
        return len(self._od)
Example #9
0
class DownloaderSimulator(BaseDownloaderSimulator):
    def __init__(self, rate):
        self._requests_per_slot = rate
        self.slots = OrderedDict()
        super(DownloaderSimulator, self).__init__()

    def update(self, requests):
        for request in requests:
            hostname = urlparse(request.url).hostname or ""
            self.slots.setdefault(hostname, deque()).append(request)

    def download(self):
        output = []
        _trash_can = []
        for key, requests in self.slots.iteritems():
            for i in range(min(len(requests), self._requests_per_slot)):
                output.append(requests.popleft())
            if not requests:
                _trash_can.append(key)

        for key in _trash_can:
            del self.slots[key]
        return output

    def downloader_info(self):
        info = {"key_type": "domain", "overused_keys": []}
        for key, requests in self.slots.iteritems():
            if len(requests) > self._requests_per_slot:
                info["overused_keys"].append(key)
        return info

    def idle(self):
        return len(self.slots) == 0
Example #10
0
    def _blockData(self, data_1, data_2):

        blocks = OrderedDict({})
        coverage = {}

        for field in self.blocker.tfidf_fields:
            fields_1 = ((record_id, record[field]) for record_id, record in data_1.iteritems())
            fields_2 = ((record_id, record[field]) for record_id, record in data_2.iteritems())

            self.blocker.tfIdfBlock(fields_1, fields_2, field)

        for block_key, record_id in self.blocker(data_1.iteritems()):
            blocks.setdefault(block_key, ([], []))[0].append((record_id, data_1[record_id]))

        for block_key, record_id in self.blocker(data_2.iteritems()):
            if block_key in blocks:
                blocks[block_key][1].append((record_id, data_2[record_id]))

        for block_id, (block, sources) in enumerate(blocks.iteritems()):
            for source in sources:
                for record_id, record in source:
                    coverage.setdefault(record_id, []).append(block_id)

        for block_id, (block_key, sources) in enumerate(blocks.iteritems()):
            tuple_block = []
            for source in sources:
                tuple_source = []
                for record_id, record in source:
                    smaller_ids = set([covered_id for covered_id in coverage[record_id] if covered_id < block_id])
                    tuple_source.append((record_id, record, smaller_ids))
                tuple_block.append(tuple_source)

            yield tuple_block
Example #11
0
def _simpleconfig_normalize(me, parent_section):
    # must not attempt to modify anything from parent_section in-place
    parent_tag, parent_options, parent_sections = parent_section
    parent_sections_new = []
    for s in parent_sections:
        tag, options, sections = s
        opts = OrderedDict()
        for n, v in options:
            vals = opts.setdefault(n, [])
            if v in vals:
                log.warning("omitting duplicated `({0}, {1}}' option from" " normalization".format(n, v))
            else:
                vals.append(v)
        options = []
        for n, vals in opts.iteritems():
            options.append((n, vals[0]))
            vals[:] = vals[1:]
        parent_sections_new.append((tag, options, [me(s) for s in sections]))
        for i, (n, vals) in enumerate((n, vals) for n, vals in opts.iteritems() if vals):
            if not i and sections:
                log.warning(
                    "current section `{0}' needs normalization but" " contains subsections (not expected)".format(tag)
                )
            for v in vals:
                parent_sections_new.append((tag, ((n, v),), ()))
    return (parent_tag, parent_options, parent_sections_new)
Example #12
0
def from_xpaths(container, xpaths):
    """
    Generate a Table of Contents from a list of XPath expressions. Each
    expression in the list corresponds to a level of the generate ToC. For
    example: :code:`['//h:h1', '//h:h2', '//h:h3']` will generate a three level
    table of contents from the ``<h1>``, ``<h2>`` and ``<h3>`` tags.
    """
    tocroot = TOC()
    xpaths = [XPath(xp) for xp in xpaths]
    level_prev = {i + 1: None for i in xrange(len(xpaths))}
    level_prev[0] = tocroot

    # Find those levels that have no elements in all spine items
    maps = OrderedDict()
    empty_levels = {i + 1 for i, xp in enumerate(xpaths)}
    for spinepath in container.spine_items:
        name = container.abspath_to_name(spinepath)
        root = container.parsed(name)
        level_item_map = maps[name] = {i + 1: frozenset(xp(root)) for i, xp in enumerate(xpaths)}
        for lvl, elems in level_item_map.iteritems():
            if elems:
                empty_levels.discard(lvl)
    # Remove empty levels from all level_maps
    if empty_levels:
        for name, lmap in tuple(maps.iteritems()):
            lmap = {lvl: items for lvl, items in lmap.iteritems() if lvl not in empty_levels}
            lmap = sorted(lmap.iteritems(), key=itemgetter(0))
            lmap = {i + 1: items for i, (l, items) in enumerate(lmap)}
            maps[name] = lmap

    for name, level_item_map in maps.iteritems():
        root = container.parsed(name)
        item_level_map = {e: i for i, elems in level_item_map.iteritems() for e in elems}
        item_dirtied = False

        for item in root.iterdescendants(etree.Element):
            lvl = plvl = item_level_map.get(item, None)
            if lvl is None:
                continue
            parent = None
            while parent is None:
                plvl -= 1
                parent = level_prev[plvl]
            lvl = plvl + 1
            if item_at_top(item):
                dirtied, elem_id = False, None
            else:
                dirtied, elem_id = ensure_id(item)
            text = elem_to_toc_text(item)
            item_dirtied = dirtied or item_dirtied
            toc = parent.add(text, name, elem_id)
            toc.dest_exists = True
            level_prev[lvl] = toc
            for i in xrange(lvl + 1, len(xpaths) + 1):
                level_prev[i] = None

        if item_dirtied:
            container.commit_item(name, keep_parsed=True)

    return tocroot
Example #13
0
def stream_tree_top(streams):
    """
    ' This function should create the top level tree structure for the datastreams, and return it as a dictionary.
    ' Only the keys should really be necessary, though.
    '
    ' Keyword Args:
    '    streams - An iterable of datastreams.
    '
    ' Returns:
    '    A dictionary containing the top level of the tree as strings.  The values are True for nodes, and
    '    False for leaves.
    """
    nodes = {}
    leaves = {}
    for s in streams:
        # Assume for now that names are unique.
        # TODO: Validate (in save/models, etc.) that no 2 objects can have names such that name and name::other
        #  exist.
        spart = s.name.partition("|")
        if spart[2] != "":
            if spart[0] not in nodes:
                nodes[spart[0]] = None
        else:
            leaves[spart[0]] = s.id

    nodes = OrderedDict(sorted(nodes.iteritems(), key=lambda t: t[0]))
    leaves = OrderedDict(sorted(leaves.iteritems(), key=lambda t: t[0]))
    return {"nodes": nodes, "leaves": leaves}
Example #14
0
def main(args):
    fd = fastq_to_dict(args.fastq)
    # longest sequence first
    fd = OrderedDict(sorted(fd.items(), key=lambda (k, v): len(v["seq"]), reverse=True))
    seen = set()
    ignore = set()
    i = 1
    for t_name, target in fd.iteritems():
        if i % 10 == 0:
            print >> sys.stderr, ">> processed %d reads..." % i
        seen.add(t_name)
        i += 1
        # skip reads that have already been determined as subsequences
        if t_name in ignore:
            continue
        t_id, t_cregion, t_fwork = t_name.split(":")
        for q_name, query in fd.iteritems():
            if q_name in seen or q_name in ignore:
                continue
            q_id, q_cregion, q_fwork = q_name.split(":")
            # only attempt to collapse things of the same c-region and framework
            if t_cregion != q_cregion and t_fwork != q_fwork:
                continue
            if distance(target["seq"], query["seq"]) < args.mismatches:
                ignore.add(q_name)
                i += 1
        print "@%s\n%s\n+\n%s" % (t_name, target["seq"], target["qual"])
class Tipalti:
    def __init__(self, payer_name, payer_id, private_key, debug=False):
        self.base_url = "https://ui.sandbox.tipalti.com" if debug else "https://ui.tipalti.com"

        self.params = OrderedDict()
        self.params["payer"] = payer_name
        self.params["idap"] = payer_id
        self.private_key = private_key

        self.user_params = OrderedDict()

    def add_param(self, param, value):
        self.user_params[param] = value

    def get_url(self, page_to_open):
        self.params["ts"] = int(time())
        uri = "{0}%s?{1}" % page_to_open
        res = uri.format(self.base_url, self._query_string())
        self.user_params = OrderedDict()
        return res

    def _query_string(self):
        params = OrderedDict()
        chn = chain(self.params.iteritems(), self.user_params.iteritems())
        for param, value in chn:
            params[param] = value
        hash_key = self._hash(urlencode(params), self.private_key)
        return urlencode(OrderedDict(params, hashkey=hash_key))

    @staticmethod
    def _hash(value, secret):
        return hmac.new(secret, value, digestmod=hashlib.sha256).hexdigest()
Example #16
0
    def _blockData(self, data_d):

        blocks = OrderedDict({})
        coverage = {}

        for field in self.blocker.tfidf_fields:
            self.blocker.tfIdfBlock(((record_id, record[field]) for record_id, record in data_d.iteritems()), field)

        for block_key, record_id in self.blocker(data_d.iteritems()):
            blocks.setdefault(block_key, []).append((record_id, data_d[record_id]))

        # Redundant-free Comparisons from Kolb et al, "Dedoop:
        # Efficient Deduplication with Hadoop"
        # http://dbs.uni-leipzig.de/file/Dedoop.pdf
        for block_id, (block, records) in enumerate(blocks.iteritems()):
            for record_id, record in records:
                coverage.setdefault(record_id, []).append(block_id)

        for block_id, (block_key, records) in enumerate(blocks.iteritems()):
            tuple_records = []
            for record_id, record in records:
                smaller_ids = set([covered_id for covered_id in coverage[record_id] if covered_id < block_id])
                tuple_records.append((record_id, record, smaller_ids))

            yield tuple_records
Example #17
0
def output_benchmarks_exp3():
    tp_file = open("exp3_throughput.dat", "w")
    lat_file = open("exp3_delay.dat", "w")
    tp_columns = OrderedDict({second: [] for second in range(21)})
    lat_columns = OrderedDict({second: [] for second in range(21)})

    for tcp in TCP_VARIANTS:
        for queue in QUEUING_VARIANTS:
            fname = "exp3_trace/" + tcp + "_" + queue + ".tr"
            for time, tp, lat in calculate_benchmarks_exp3(fname):
                tp_columns[time].append(tp)
                lat_columns[time].append(lat)

    for time, tps in tp_columns.iteritems():
        if tps:
            tp_file.write("%d\t%.3f\t%.3f\t%.3f\t%.3f\n" % (time, tps[0], tps[1], tps[2], tps[3]))
        else:
            tp_file.write("0\t0.000\t0.000\t0.000\t0.000\n")
    for time, lats in lat_columns.iteritems():
        if lats:
            lat_file.write("%d\t%.3f\t%.3f\t%.3f\t%.3f\n" % (time, lats[0], lats[1], lats[2], lats[3]))
        else:
            lat_file.write("0\t0.000\t0.000\t0.000\t0.000\n")

    tp_file.close()
    lat_file.close()
Example #18
0
def createEnumerated(element):
    valuemap = OrderedDict()
    name = "UA_" + element.get("Name")
    fixed_size.add(name)
    for child in element:
        if child.tag == "{http://opcfoundation.org/BinarySchema/}Documentation":
            printh("/** @brief " + child.text + " */")
        if child.tag == "{http://opcfoundation.org/BinarySchema/}EnumeratedValue":
            valuemap[name + "_" + child.get("Name")] = child.get("Value")
    valuemap = OrderedDict(sorted(valuemap.iteritems(), key=lambda (k, v): int(v)))
    printh("typedef UA_Int32 " + name + ";")
    printh(
        "enum "
        + name
        + "_enum { \n\t"
        + ",\n\t".join(map(lambda (key, value): key.upper() + " = " + value, valuemap.iteritems()))
        + "\n};"
    )
    printh("UA_TYPE_PROTOTYPES (" + name + ")")
    printh("UA_TYPE_BINARY_ENCODING(" + name + ")")
    printh("UA_TYPE_XML_ENCODING(" + name + ")\n")
    printc("UA_TYPE_AS(" + name + ", UA_Int32)")
    printc("UA_TYPE_BINARY_ENCODING_AS(" + name + ", UA_Int32)")
    printc(
        """UA_TYPE_METHOD_CALCSIZEXML_NOTIMPL(%(name)s)
UA_TYPE_METHOD_ENCODEXML_NOTIMPL(%(name)s)
UA_TYPE_METHOD_DECODEXML_NOTIMPL(%(name)s\n)"""
    )
    def forwards(self, orm):
        # make sure all users have an email
        for user in orm["users.User"].objects.filter(email=""):
            user.email = user.username
            user.save()

        emails = OrderedDict()
        for userid, email in orm["users.User"].objects.values_list("id", "email"):
            emails.setdefault(email, []).append(userid)

        bad = set()
        for email, userids in emails.iteritems():
            if len(userids) > 1:
                logger.error(
                    'E-mail address "%s" belongs to multiple users '
                    "with IDs %s!" % (email, ", ".join(map(str, userids)))
                )
                bad.add(email)

        for email, userids in emails.iteritems():
            for userid in userids[1:]:
                updated = "%s_%s" % (userid, email)
                u = orm["users.User"].objects.filter(id=userid)
                u.update(email=updated)
                logger.warning("Renamed %s -> %s" % (email, updated))

        if bad:
            logger.warning("Some usernames were changed")
Example #20
0
def createStructured(element):
    valuemap = OrderedDict()
    name = element.get("Name")
    print "-- " + name

    lengthfields = set()
    for child in element:
        if child.get("LengthField"):
            lengthfields.add(child.get("LengthField"))

    for child in element:
        if child.tag == "{http://opcfoundation.org/BinarySchema/}Documentation":
            print "-- " + child.text
        elif child.tag == "{http://opcfoundation.org/BinarySchema/}Field":
            if child.get("Name") in lengthfields:
                continue
            childname = camlCase2AdaCase(child.get("Name"))
            if childname in printed_types:
                childname = childname + "_Value"  # attributes may not have the name of a type
            typename = stripTypename(child.get("TypeName"))
            if childname == "Response_Header" or childname == "Request_Header":
                continue
            if typename in indefinite_types:
                valuemap[childname] = typename + "s.Pointer"
            elif child.get("LengthField"):
                valuemap[childname] = "ListOf" + typename + ".Pointer"
            else:
                valuemap[childname] = typename

    if "Response" in name[len(name) - 9 :]:
        print ("type " + name + " is new Response_Base with "),
    elif "Request" in name[len(name) - 9 :]:
        print ("type " + name + " is new Request_Base with "),
    else:
        print ("type " + name + " is new UA_Builtin with "),
    if len(valuemap) > 0:
        print "record"
        for n, t in valuemap.iteritems():
            print n + " : " + t + ";"
        print "end record;"
    else:
        print "null record;"
    print "function NodeId_Nr(Item : in " + name + ") return UInt16 is (SID." + name + "_Id);"  # increase id by 2 to get the binary_encoding id
    if "Response" in name[len(name) - 9 :]:
        print ("function Binary_Size(Item : " + name + ") return Int32 is ( Binary_Size(Item.Response_Header)"),
    elif "Request" in name[len(name) - 9 :]:
        print ("function Binary_Size(Item : " + name + ") return Int32 is ( Binary_Size(Item.Request_Header)"),
    else:
        print ("function Binary_Size(Item : " + name + ") return Int32 is ( 0"),  # 0 for the null records
    for n, t in valuemap.iteritems():
        if t in elementary_size:
            print ("+ " + str(elementary_size[t])),
        else:
            if t in enum_types:
                print ("+ 8")  # enums are all 32 bit
            elif t.find(".Pointer") != -1 or t.find("ListOf") != -1:
                print ("+ " + t[0 : t.find(".") + 1] + "Binary_Size(Item." + n + ")"),
            else:
                print ("+ Binary_Size(Item." + n + ")"),
    print ");\n"
Example #21
0
def generate_csv_from_registrations(registrations, regform_items, special_items):
    """Generates a CSV file from a given registration list.

    :param registrations: The list of registrations to include in the file
    :param regform_items: The registration form items to be used as columns
    :param special_items: Registration form information as extra columns
    """

    field_names = ["ID", "Name"]
    special_item_mapping = OrderedDict(
        [
            ("reg_date", ("Registration date", lambda x: format_datetime(x.submitted_dt))),
            ("state", ("Registration state", lambda x: x.state.title)),
            ("price", ("Price", lambda x: x.render_price())),
            ("checked_in", ("Checked in", lambda x: "Yes" if x.checked_in else "No")),
            ("checked_in_date", ("Check-in date", lambda x: format_datetime(x.checked_in_dt) if x.checked_in else "")),
        ]
    )
    for item in regform_items:
        field_names.append("{}_{}".format(item.title.encode("utf-8"), item.id))
        if item.input_type == "accommodation":
            field_names.append("{}_{}_{}".format(item.title.encode("utf-8"), "Arrival", item.id))
            field_names.append("{}_{}_{}".format(item.title.encode("utf-8"), "Departure", item.id))
    field_names.extend(title for name, (title, fn) in special_item_mapping.iteritems() if name in special_items)
    buf = BytesIO()
    writer = csv.DictWriter(buf, fieldnames=field_names)
    writer.writeheader()
    for registration in registrations:
        data = registration.data_by_field
        registration_dict = {
            "ID": registration.friendly_id,
            "Name": "{} {}".format(registration.first_name, registration.last_name).encode("utf-8"),
        }
        for item in regform_items:
            key = "{}_{}".format(item.title.encode("utf-8"), item.id)
            if item.input_type == "accommodation":
                registration_dict[key] = _prepare_data(
                    data[item.id].friendly_data.get("choice") if item.id in data else ""
                )
                key = "{}_{}_{}".format(item.title.encode("utf-8"), "Arrival", item.id)
                arrival_date = data[item.id].friendly_data.get("arrival_date") if item.id in data else None
                registration_dict[key] = _prepare_data(format_date(arrival_date) if arrival_date else "")
                key = "{}_{}_{}".format(item.title.encode("utf-8"), "Departure", item.id)
                departure_date = data[item.id].friendly_data.get("departure_date") if item.id in data else None
                registration_dict[key] = _prepare_data(format_date(departure_date) if departure_date else "")
            else:
                registration_dict[key] = _prepare_data(data[item.id].friendly_data if item.id in data else "")
        for name, (title, fn) in special_item_mapping.iteritems():
            if name not in special_items:
                continue
            value = fn(registration)
            if isinstance(value, unicode):
                value = value.encode("utf-8")
            registration_dict[title] = value
        writer.writerow(registration_dict)
    buf.seek(0)
    return buf
Example #22
0
class matrixStatistics(matrixCreate):

    def __init__(self, rows, cols, data=None, name=None, rowname='Observations', colname='Features'):
        matrixCreate.__init__(self, rows=rows, cols=cols, data=data, name=name, rowname=rowname, colname=colname)
        self.buildMatrix()
        self.hasComputedLargestFeatures=False
        self.hasComputedLargestObservations=False

    def _rowSums(self):
        self.rowSums=np.asarray(self.matrix.sum(axis=1).T)[0]

    def _largestObservations(self):
        self._rowSums()
        tops = np.argsort(self.rowSums)[::-1]
        obs = np.asarray([self.rowDict[topIndex] for topIndex in tops])
        self.observationSumsDict = OrderedDict(zip(obs, self.rowSums[tops]))
        self.rowTopIndices = tops
        self.hasComputedLargestObservations=True

    def _colSums(self):
        self.colSums=np.asarray(self.matrix.sum(axis=0))[0]

    def _largestFeatures(self):
        self._colSums()
        tops = np.argsort(self.colSums)[::-1]
        features = np.asarray([self.colDict[topIndex] for topIndex in tops])
        self.featureSumsDict = OrderedDict(zip(features, self.colSums[tops]))
        self.colTopIndices = tops
        self.hasComputedLargestFeatures=True

    def _computeLargest(self):
        self._largestObservations()
        self._largestFeatures()

    def describeFeatures(self, topN=10):
        if not self.hasComputedLargestFeatures:
            self._largestFeatures()
        i =0
        print "-"*40
        for k,v in self.featureSumsDict.iteritems():
            print "%s %s has %s nonzero values across the %s"%(self.namecol, str(k), str(int(v)), self.namerow)
            i+=1
            if i>topN:
                break

    def describeObservations(self, topN=10):
        if not self.hasComputedLargestObservations:
            self._largestObservations()
        i =0
        print "-"*40
        for k,v in self.observationSumsDict.iteritems():
            print "%s %s has %s nonzero values across the %s"%(self.namerow, str(k), str(int(v)), self.namecol)
            i+=1
            if i>topN:
                break
Example #23
0
class Tweak(object):  # {{{
    def __init__(self, name, doc, var_names, defaults, custom):
        translate = _
        self.name = translate(name)
        self.doc = doc.strip()
        if self.doc:
            self.doc = translate(self.doc)
        self.var_names = var_names
        if self.var_names:
            self.doc = u"%s: %s\n\n%s" % (_("ID"), self.var_names[0], self.doc)
        self.default_values = OrderedDict()
        for x in var_names:
            self.default_values[x] = defaults[x]
        self.custom_values = OrderedDict()
        for x in var_names:
            if x in custom:
                self.custom_values[x] = custom[x]

    def __str__(self):
        ans = ["#: " + self.name]
        for line in self.doc.splitlines():
            if line:
                ans.append("# " + line)
        for key, val in self.default_values.iteritems():
            val = self.custom_values.get(key, val)
            ans.append("%s = %r" % (key, val))
        ans = "\n".join(ans)
        if isinstance(ans, unicode):
            ans = ans.encode("utf-8")
        return ans

    def __cmp__(self, other):
        return -1 * cmp(self.is_customized, getattr(other, "is_customized", False))

    @property
    def is_customized(self):
        for x, val in self.default_values.iteritems():
            if self.custom_values.get(x, val) != val:
                return True
        return False

    @property
    def edit_text(self):
        ans = ["# %s" % self.name]
        for x, val in self.default_values.iteritems():
            val = self.custom_values.get(x, val)
            ans.append("%s = %r" % (x, val))
        return "\n\n".join(ans)

    def restore_to_default(self):
        self.custom_values.clear()

    def update(self, varmap):
        self.custom_values.update(varmap)
Example #24
0
def generate_spreadsheet_from_abstracts(abstracts, static_item_ids, dynamic_items):
    """Generates a spreadsheet data from a given abstract list.

    :param abstracts: The list of abstracts to include in the file
    :param static_item_ids: The abstract properties to be used as columns
    :param dynamic_items: Contribution fields as extra columns
    """
    field_names = ["Id", "Title"]
    static_item_mapping = OrderedDict(
        [
            ("state", ("State", lambda x: x.state.title)),
            ("submitter", ("Submitter", lambda x: x.submitter.full_name)),
            ("authors", ("Primary authors", lambda x: [a.full_name for a in x.primary_authors])),
            (
                "accepted_track",
                ("Accepted track", lambda x: x.accepted_track.short_title if x.accepted_track else None),
            ),
            (
                "submitted_for_tracks",
                ("Submitted for tracks", lambda x: [t.short_title for t in x.submitted_for_tracks]),
            ),
            ("reviewed_for_tracks", ("Reviewed for tracks", lambda x: [t.short_title for t in x.reviewed_for_tracks])),
            (
                "accepted_contrib_type",
                ("Accepted type", lambda x: x.accepted_contrib_type.name if x.accepted_contrib_type else None),
            ),
            (
                "submitted_contrib_type",
                ("Submitted type", lambda x: x.submitted_contrib_type.name if x.submitted_contrib_type else None),
            ),
            ("score", ("Score", lambda x: round(x.score, 1))),
            ("submitted_dt", ("Submission date", lambda x: to_unicode(format_datetime(x.submitted_dt)))),
            (
                "modified_dt",
                ("Modification date", lambda x: (to_unicode(format_datetime(x.modified_dt)) if x.modified_dt else "")),
            ),
        ]
    )
    field_names.extend(unique_col(item.title, item.id) for item in dynamic_items)
    field_names.extend(title for name, (title, fn) in static_item_mapping.iteritems() if name in static_item_ids)
    rows = []
    for abstract in abstracts:
        data = abstract.data_by_field
        abstract_dict = {"Id": abstract.friendly_id, "Title": abstract.title}
        for item in dynamic_items:
            key = unique_col(item.title, item.id)
            abstract_dict[key] = data[item.id].friendly_data if item.id in data else ""
        for name, (title, fn) in static_item_mapping.iteritems():
            if name not in static_item_ids:
                continue
            value = fn(abstract)
            abstract_dict[title] = value
        rows.append(abstract_dict)
    return field_names, rows
Example #25
0
def generate_spreadsheet_from_registrations(registrations, regform_items, special_items):
    """Generates a spreadsheet data from a given registration list.

    :param registrations: The list of registrations to include in the file
    :param regform_items: The registration form items to be used as columns
    :param special_items: Registration form information as extra columns
    """
    field_names = ["ID", "Name"]
    special_item_mapping = OrderedDict(
        [
            ("reg_date", ("Registration date", lambda x: to_unicode(format_datetime(x.submitted_dt)))),
            ("state", ("Registration state", lambda x: x.state.title)),
            ("price", ("Price", lambda x: x.render_price())),
            ("checked_in", ("Checked in", lambda x: x.checked_in)),
            (
                "checked_in_date",
                ("Check-in date", lambda x: (to_unicode(format_datetime(x.checked_in_dt)) if x.checked_in else "")),
            ),
        ]
    )
    for item in regform_items:
        field_names.append("{}_{}".format(item.title, item.id))
        if item.input_type == "accommodation":
            field_names.append("{}_{}_{}".format(item.title, "Arrival", item.id))
            field_names.append("{}_{}_{}".format(item.title, "Departure", item.id))
    field_names.extend(title for name, (title, fn) in special_item_mapping.iteritems() if name in special_items)
    rows = []
    for registration in registrations:
        data = registration.data_by_field
        registration_dict = {
            "ID": registration.friendly_id,
            "Name": "{} {}".format(registration.first_name, registration.last_name),
        }
        for item in regform_items:
            key = "{}_{}".format(item.title, item.id)
            if item.input_type == "accommodation":
                registration_dict[key] = _prepare_data(
                    data[item.id].friendly_data.get("choice") if item.id in data else ""
                )
                key = "{}_{}_{}".format(item.title, "Arrival", item.id)
                arrival_date = data[item.id].friendly_data.get("arrival_date") if item.id in data else None
                registration_dict[key] = _prepare_data(format_date(arrival_date) if arrival_date else "")
                key = "{}_{}_{}".format(item.title, "Departure", item.id)
                departure_date = data[item.id].friendly_data.get("departure_date") if item.id in data else None
                registration_dict[key] = _prepare_data(format_date(departure_date) if departure_date else "")
            else:
                registration_dict[key] = _prepare_data(data[item.id].friendly_data if item.id in data else "")
        for name, (title, fn) in special_item_mapping.iteritems():
            if name not in special_items:
                continue
            value = fn(registration)
            registration_dict[title] = value
        rows.append(registration_dict)
    return field_names, rows
Example #26
0
 def _perform_workflow(data):
     tn, f = data
     if "_blast.out" in f:
         genome_specific_dict = {}
         name = get_seq_name(f)
         reduced_name = name.replace(".fasta.new_blast.out", "")
         genome_specific_dict.update({"ID": reduced_name})
         outfile = open("%s.counts.txt" % reduced_name, "w")
         try:
             for line in open(f, "U"):
                 newline = line.strip()
                 fields = newline.split()
                 """Each blast query should be in the reference blast file"""
                 if fields[0] not in ref_scores:
                     print("potential problem found with BLAST File..")
                     sys.exit()
                 elif float(fields[2]) >= int(min_hlog) and (
                     float(fields[11]) / float(ref_scores.get(fields[0]))
                 ) >= float(length):
                     try:
                         my_dict_o[fields[0]].append(fields[11])
                         genome_specific_dict[fields[0]].append(fields[11])
                     except KeyError:
                         my_dict_o[fields[0]] = [fields[11]]
                         genome_specific_dict[fields[0]] = [fields[11]]
                 else:
                     continue
         except:
             raise TypeError("problem parsing %s" % f)
         new_dict = {}
         for k, v in genome_specific_dict.iteritems():
             for cluster in clusters:
                 if k == "ID":
                     pass
                 elif k == cluster:
                     try:
                         new_dict.update({k: len(v)})
                     except:
                         new_dict.update({k: "0"})
         for cluster in clusters:
             if cluster not in genome_specific_dict:
                 new_dict.update({cluster: "0"})
         """this is our ordered dictionary"""
         od = collections.OrderedDict(sorted(new_dict.items()))
         ids = OrderedDict({"ID": reduced_name})
         both = OrderedDict(list(ids.items()) + list(new_dict.items()))
         for k, v in both.iteritems():
             if k == "ID":
                 outfile.write(str(v) + "\n")
         for cluster in clusters:
             for k, v in both.iteritems():
                 if k == cluster:
                     outfile.write(str(v) + "\n")
         outfile.close()
Example #27
0
def get_transcripts_simple(db, location=None, use_strand=False, overlap=True):
    sql, sql_parameters = location_to_sql(location, use_strand, overlap)
    sql = "{0} {1} {2}".format(SQL_TRANSCRIPTS_SIMPLE, sql, SQL_TRANSCRIPTS_SIMPLE_ORDER_BY)

    LOG.debug("SQL:\n{0}".format(sql))
    LOG.debug("PARAMETERS: {0}".format(sql_parameters))

    conn = sqlite3.connect(db)
    sqlite3.enable_callback_tracebacks(True)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()

    cursor.execute(sql, sql_parameters)

    transcripts = OrderedDict()
    exons = OrderedDict()
    for r in cursor:

        if r["transcript_id"] == r["child_id"]:
            # transcript
            if r["_key"] not in transcripts:
                transcripts[r["_key"]] = Transcript(
                    r["transcript_id"],
                    r["transcript_seqid"],
                    r["transcript_start"],
                    r["transcript_end"],
                    r["transcript_strand"],
                )
        else:
            # exon
            exon = exons.get(
                r["_child_key"],
                Exon(r["child_id"], r["child_seqid"], r["child_start"], r["child_end"], r["child_strand"]),
            )
            exon.gene_id = r["transcript_id"]
            exon.transcript_ids[r["transcript_id"]] = r["transcript_id"]
            attribute = r["gtf_attribute"]
            value = r["value"]

            if attribute == "exon_number":
                exon.exon_number = value

            exons[r["_child_key"]] = exon

    transcripts = {transcript.ensembl_id: transcript for i, transcript in transcripts.iteritems()}

    for _id, exon in exons.iteritems():
        for _tid in exon.transcript_ids:
            transcripts[_tid].exons[exon.ensembl_id] = exon

    cursor.close()
    conn.close()

    return transcripts.values()
class RepositoryHandler:
    def __init__(self, working_directory, logger):
        self.logger = logger
        self.working_directory = working_directory
        self.repo_list_file = working_directory + "repo_list.json"
        self.repo_status_file = working_directory + "repo_status.json"
        self.repo_list = OrderedDict()
        self.create_repo_list_and_status_from_files()
        self.logger.debug("Repository handler started")

    def create_repo_list_and_status_from_files(self):
        repo_list = self.load_repo_list_from_file()
        for repo_id, repo_data in sorted(repo_list.iteritems(), key=lambda r: r[1]["name"]):
            try:
                self.repo_list[repo_id] = Repository(repo_id, repo_list[repo_id], self.working_directory, self.logger)
            except KeyError:
                self.logger.exception("Got KeyError during Repository instantiation.")

    def load_status_info_from_file(self):
        repo_status_info = self.load_repo_status_from_file()
        for repo_id, repo_data in self.repo_list.iteritems():
            if repo_status_info and repo_id in repo_status_info:
                self.get_repo_by_id(repo_id).add_status_info_from_json(repo_status_info[repo_id])

    def get_repo_list(self):
        return self.repo_list.values()

    def get_repo_by_id(self, repo_id):
        return self.repo_list[repo_id]

    def load_repo_status_from_file(self):
        try:
            with open(self.repo_status_file) as repo_status:
                return json.load(repo_status)
        except IOError:
            self.logger.info("repo status file %s doesn't exist" % self.repo_status_file)
            return {}

    def load_repo_list_from_file(self):
        try:
            with open(self.repo_list_file) as repo_list:
                return json.load(repo_list)
        except IOError:
            self.logger.critical("repo list file %s doesn't exist" % self.repo_list_file)
            return {}

    def save_repo_status_to_file(self):
        if not self.repo_list:
            self.logger.warning("Got empty repository list, not updating status file!")
            return

        with open(self.repo_status_file, "w") as repo_status:
            json.dump({k: v.to_dict() for k, v in self.repo_list.iteritems()}, repo_status, indent=4, sort_keys=True)
Example #29
0
 def list(self, full=False, verbose=False):
     quiet = not verbose
     response, content = self.api.get(self.root_url, quiet=quiet)
     content = content or {}
     ordered = OrderedDict(sorted(content.items(), key=lambda x: x[1]["requested"]))
     if not full:
         for uid, data in ordered.iteritems():
             self.log.info(" • {} <{}> [{} - {}]".format(uid, data["command"], data["requested"], data["status"]))
     else:
         for uid, data in ordered.iteritems():
             self.log.info(" • {}:".format(uid))
             self.print_info(data, prompt="   ° ")
Example #30
0
class IndexedPQ(object):
    def __init__(self):
        super(IndexedPQ, self).__init__()
        self.ipq = OrderedDict()

    def isEmpty(self):
        return len(self.ipq) == 0

    def size(self):
        return len(self.ipq)

    def insert(self, key, val):
        self.ipq[key] = val
        self.ipq = OrderedDict(sorted(self.ipq.iteritems(), cmp=self._cmp, key=lambda d: d))

    def change(self, key, val):
        if not key in self.ipq:
            raise ValueError("Invalid key: %s" % key)
        else:
            self.ipq[key] = val
            self.ipq = OrderedDict(sorted(self.ipq.iteritems(), cmp=self._cmp, key=lambda d: d))

    def delete(self, key):
        del self.ipq[key]

    def minIndex(self):
        return self.ipq.keys()[0]

    def min(self):
        return self.values()[0]

    def delMin(self):
        indexOfMin = self.ipq.keys()[0]
        del self.ipq[indexOfMin]
        return indexOfMin

    def show(self):
        return self.ipq.items()

    def keys(self):
        return self.ipq.keys()

    def _cmp(self, a, b):
        if a[1] < b[1]:
            return -1
        elif a[1] > b[1]:
            return 1
        else:
            if a[0] < b[0]:
                return -1
            else:
                return 1