def link_regions_to_fed(args):

    #try:
    db_session = DBSession()
    transaction.manager.begin()

    # 1. Get all federals
    fds = db_session.query(FederalDistrict).all()

    # 2. Get all regions
    regions = db_session.query(Region).all()

    # 3. Read csv file with federals (for get original fed id)
    csv_fds = get_from_csv(path.join(BASE_PATH, 'federal_districts.csv'))

    # 4. Read updated csv file with regions and federal ids
    csv_regions = get_from_csv(path.join(BASE_PATH, 'regions.csv'))

    # 5. Update regions in DB
    for region in regions:
        # get fed_id from csv by region_code
        orig_fed_id = next(ifilter(lambda x: x['region_code'] == str(region.region_code), csv_regions))['fed_id']
        # get original federal from csv
        orig_fed = next(ifilter(lambda x: x['id'] == orig_fed_id, csv_fds))
        # find federal in db by short_name
        db_fed = next(ifilter(lambda fed: fed.short_name == unicode(orig_fed['short_name'], 'utf8'), fds))
        # update db region
        region.federal_dist = db_fed

    transaction.manager.commit()
    db_session.close()

    print ('Region was linked with federal districts')
Ejemplo n.º 2
0
 def edges(self, src_node=None, dst_node=None):
     _edges = self._edges
     if src_node:
         _edges = ifilter(lambda x: x.source == src_node.id, _edges)
     if dst_node:
         _edges = ifilter(lambda x: x.target == dst_node.id, _edges)
     return _edges
Ejemplo n.º 3
0
    def _guess_cover(self, files):
        """Return the filename within <files> that is the most likely to be the
        cover of an archive using some simple heuristics.
        """
        # Ignore MacOSX meta files.
        files = itertools.ifilter(lambda filename:
                u'__MACOSX' not in os.path.normpath(filename).split(os.sep),
                files)
        # Ignore credit files if possible.
        files = itertools.ifilter(lambda filename:
                u'credit' not in os.path.split(filename)[1].lower(), files)

        images = list(itertools.ifilter(image_tools.is_image_file, files))

        tools.alphanumeric_sort(images)

        front_re = re.compile('(cover|front)', re.I)
        candidates = filter(front_re.search, images)
        candidates = [c for c in candidates if 'back' not in c.lower()]

        if candidates:
            return candidates[0]

        if images:
            return images[0]

        return None
Ejemplo n.º 4
0
def get_containers(template):
		# Build a tree of the templates we're using, placing the root template first.
		levels = build_extension_tree(template.nodelist)
		
		contentlet_specs = []
		contentreference_specs = SortedDict()
		blocks = {}
		
		for level in reversed(levels):
			level.initialize()
			contentlet_specs.extend(itertools.ifilter(lambda x: x not in contentlet_specs, level.contentlet_specs))
			contentreference_specs.update(level.contentreference_specs)
			for name, block in level.blocks.items():
				if block.block_super:
					blocks.setdefault(name, []).append(block)
				else:
					blocks[name] = [block]
		
		for block_list in blocks.values():
			for block in block_list:
				block.initialize()
				contentlet_specs.extend(itertools.ifilter(lambda x: x not in contentlet_specs, block.contentlet_specs))
				contentreference_specs.update(block.contentreference_specs)
		
		return contentlet_specs, contentreference_specs
Ejemplo n.º 5
0
def ignore(context):
    '''.ignore nick!user@host'''
    bot.config.setdefault('IGNORE', [])

    if context.args:
        to_ignore = glob(context.args)
        supersets = list(ifilter(lambda ignored: to_ignore.issub(glob(ignored)), bot.config['IGNORE']))
        if len(supersets) > 0:
            return 'Not ignoring \x02%s\x02 because it is already matched by \x02%s\x02' % (context.args, supersets[0])

        filter = lambda ignored: to_ignore.issuper(glob(ignored))
        removed = list(ifilter(filter, bot.config['IGNORE']))

        bot.config['IGNORE'] = list(ifilterfalse(filter, bot.config['IGNORE']))
        bot.config['IGNORE'].append(context.args)

        save_ignores()

        bot.log(context, ('IGNORE'), '+{0}{1}'.format(context.args, (' -' + ' -'.join(removed)) if removed else ''))

        if removed:
            return 'Ignored and removed \x02%d\x02 redundant ignores: \x02%s\x02' % (len(removed), '\x02, \x02'.join(removed))
        else:
            return 'Ignored.'
    else:
        return eval.__doc__
Ejemplo n.º 6
0
def master_up(key_name, credential_file="~/.rackspace_cloud_credentials"):
    '''
    Create a salt-master on Rackspace

    Alternatively create the master using nova
    '''

    # Authenticate with Rackspace, use credential file
    pyrax.set_setting("identity_type", "rackspace")
    pyrax.set_credential_file(os.path.expanduser(credential_file))

    # Shorthand
    cs = pyrax.cloudservers

    # Building Ubuntu 12.04 boxes with 512 MB RAM
    iter_flavors = ifilter(lambda flavor: flavor.ram == 512, cs.flavors.list())
    flavor_512 = iter_flavors.next()
    iter_os = ifilter(lambda img: "Ubuntu 12.04" in img.name, cs.images.list())
    ubu_image = iter_os.next()

    master = cs.servers.create("master.ipython.org", ubu_image.id, flavor_512,
                               key_name=key_name)

    master = pyrax.utils.wait_for_build(master, verbose=True)

    env.hosts = [master.accessIPv4]

    print("Master IP: {}".format(master.accessIPv4))
    return master.accessIPv4
Ejemplo n.º 7
0
    def _resolveSpecialSegment(self, segmentB, specialResolutionMethods):
        resolutionMethodExecutor = _compileRules(specialResolutionMethods, 3)
        for (resolutionMethod, args) in resolutionMethodExecutor[True]:
            iterables = []
            for arg in args:
                iterables.append(itertools.repeat(arg))
            resolutions = itertools.imap(resolutionMethod, self.allCorrectSinglePossibilities(), *iterables)
            correctAB = itertools.izip(self.allCorrectSinglePossibilities(), resolutions)
            correctAB = itertools.ifilter(
                lambda possibAB: possibility.pitchesWithinLimit(possibA=possibAB[1], maxPitch=segmentB._maxPitch),
                correctAB,
            )
            if self.fbRules.applyConsecutivePossibRulesToResolution:
                correctAB = itertools.ifilter(
                    lambda possibAB: self._isCorrectConsecutivePossibility(possibA=possibAB[0], possibB=possibAB[1]),
                    correctAB,
                )
            if self.fbRules.applySinglePossibRulesToResolution:
                segmentB._singlePossibilityRuleChecking = _compileRules(
                    segmentB.singlePossibilityRules(segmentB.fbRules)
                )
                correctAB = itertools.ifilter(
                    lambda possibAB: segmentB._isCorrectSinglePossibility(possibA=possibAB[1]), correctAB
                )
            return correctAB

        raise SegmentException("No standard resolution available.")
Ejemplo n.º 8
0
def ts_guess_manifest_v1(ts_path):
    """Guesses the values of manifest fields in a timestream
    """
    # This whole thing's one massive f*****g kludge. But it seems to work
    # pretty good so, well, whoop.
    retval = {}
    # get a sorted list of all files
    all_files = []
    for root, folders, files in os.walk(ts_path):
        for folder in folders:
            if folder.startswith("_"):
                folders.remove(folder)
        for fle in files:
            all_files.append(path.join(root, fle))
    all_files = sorted(all_files)
    # find most common extension, and assume this is the ext
    exts = collections.Counter(IMAGE_EXT_CONSTANTS)
    our_exts = map(lambda x: path.splitext(x)[1][1:], all_files)
    our_exts = ifilter(lambda x: x.lower() in IMAGE_EXT_CONSTANTS, our_exts)
    for ext in our_exts:
        try:
            exts[ext] += 1
        except KeyError:
            pass
    # most common gives list of tuples. [0] = (ext, count), [0][0] = ext
    retval["extension"] = exts.most_common(1)[0][0]
    all_files = ifilter(
        lambda x: path.splitext(x)[1][1:] == retval["extension"],
        all_files)
    # get image type from extension:
    try:
        retval["image_type"] = IMAGE_EXT_TO_TYPE[retval["extension"]]
    except KeyError:
        retval["image_type"] = None
    # Get list of images:
    images = ifilter(
        lambda x: path.splitext(x)[1][1:] == retval["extension"],
        all_files)
    # decode times from images:
    times = map(ts_parse_date_path, sorted(images))
    # get first and last dates:
    try:
        retval["start_datetime"] = ts_format_date(times[0])
        retval["end_datetime"] = ts_format_date(times[-1])
    except IndexError:
        msg = "{} is an invalid V1 timestream".format(ts_path)
        LOG.error(msg)
        raise ValueError(msg)
    # Get time intervals between images
    intervals = list()
    for iii in range(len(times) - 1):
        interval = times[iii + 1] - times[iii]
        intervals.append(interval.seconds)
    retval["interval"] = max(min(intervals), 1)
    retval["name"] = path.basename(ts_path.rstrip(os.sep))
    # This is dodgy isn't it :S
    retval["missing"] = []
    # If any of this worked, it must be version 1
    retval["version"] = 1
    return retval
Ejemplo n.º 9
0
    def _analyseRepository(self, repositoryName, dateSince):
        reportfile = os.path.join(self.eventpath, repositoryName + ".events")

        records, seconds = 0, 0.0
        if not isfile(reportfile):
            return records, seconds
        events = open(reportfile)
        try:
            split = lambda l: map(str.strip, l.split("\t"))
            begintime = None
            datefilter = lambda (date, x, y, z): date[1:-1] >= dateSince
            allevents = imap(split, ifilter(str.strip, events))
            for date, event, anIdentifier, comments in ifilter(datefilter, allevents):
                if event == "STARTHARVEST":
                    begintime = parseToTime(date[1:-1])
                    harvested = uploaded = deleted = total = -1
                elif event == "ENDHARVEST":
                    if begintime and harvested > -1:
                        endtime = parseToTime(date[1:-1])
                        if endtime > begintime:
                            records += int(uploaded) + int(deleted)
                            seconds += diffTime(endtime, begintime)
                            begintime = None
                elif event == "SUCCES":
                    match = NUMBERS_RE.match(comments)
                    if match:
                        harvested, uploaded, deleted, total = match.groups()

        finally:
            events.close()
        return records, seconds
Ejemplo n.º 10
0
def PosSecStruc(Pairs, PairNum):
    
    # discards structures in which one base pairs  more than one time
    def DuplicatesCheck(p):
        subcharL = []
        for char in p:
            for subchar in char:
                subcharL.append(subchar)
        for subchar in subcharL:        
            if subcharL.count(subchar) > 1:
                    return False

        return True
        
    #discardes structures in which the basepairs would overlap
    def OverlapCheck(ps):
        struc = sorted(ps)
        for i in xrange(len(struc)):
            for j in xrange(i +1,len(struc)):
                if struc[i][0] < struc[j][0] < struc[i][1] < struc[j][1]:
                    return False
                if struc[j][0] < struc[i][0] < struc[j][1] < struc[i][1]:
                    return False
        return True
            
    # this is very efficient code!!!!!! <--- REUSE        
    strucs = list(ifilter(lambda x: DuplicatesCheck(x), list(combinations(Pairs, PairNum))))
    out = list(ifilter(lambda x: OverlapCheck(x), strucs))
    return out
Ejemplo n.º 11
0
 def iteritems(self):
     definitions = type(self).configuration_setting_definitions
     version = self.command.protocol_version
     return ifilter(
         lambda (name, value): value is not None, imap(
             lambda setting: (setting.name, setting.__get__(self)), ifilter(
                 lambda setting: setting.is_supported_by_protocol(version), definitions)))
Ejemplo n.º 12
0
    def everything(cls, *tags, **location):
        """Read all of the specified `tags` within the database using the cache.

        Returns a tuple of the format `(Globals, Contents, Frames)`. Each field
        is a dictionary keyed by location or offset that retains the tags that
        were read. If the boolean `location` was specified then key each
        contents tag by location instead of address.
        """
        global export

        # collect all the globals into a dictionary
        six.print_(u'--> Grabbing globals (cached)...', file=output)
        iterable = export.globals(*tags)
        Globals = {ea : res for ea, res in itertools.ifilter(None, iterable)}

        # grab all the contents into a dictionary
        six.print_(u'--> Grabbing contents from functions (cached)...', file=output)
        location = location.get('location', False)
        iterable = export.contents(*tags, location=location)
        Contents = {loc : res for loc, res in itertools.ifilter(None, iterable)}

        # grab any frames into a dictionary
        six.print_(u'--> Grabbing frames from functions (cached)...', file=output)
        iterable = export.frames(*tags)
        Frames = {ea : res for ea, res in itertools.ifilter(None, iterable)}

        # return it back to the user
        return Globals, Contents, Frames
Ejemplo n.º 13
0
def ignore(context):
    """.ignore nick!user@host"""
    bot.config.setdefault("IGNORE", [])
    if not utils.isadmin(context.line["prefix"], bot):
        return
    if context.args:
        to_ignore = glob(context.args)
        supersets = list(ifilter(lambda ignored: to_ignore.issub(glob(ignored)), bot.config["IGNORE"]))
        if len(supersets) > 0:
            return "Not ignoring \x02%s\x02 because it is already matched by \x02%s\x02" % (context.args, supersets[0])

        filter = lambda ignored: to_ignore.issuper(glob(ignored))
        removed = list(ifilter(filter, bot.config["IGNORE"]))

        bot.config["IGNORE"] = list(ifilterfalse(filter, bot.config["IGNORE"]))
        bot.config["IGNORE"].append(context.args)

        save_ignores()

        if len(removed) > 0:
            return "Ignored and removed \x02%d\x02 redundant ignores: \x02%s\x02" % (
                len(removed),
                "\x02, \x02".join(removed),
            )
        else:
            return "Ignored."
    else:
        return eval.__doc__
Ejemplo n.º 14
0
    def containers(self):
        """
		Returns a tuple where the first item is a list of names of contentlets referenced by containers, and the second item is a list of tuples of names and contenttypes of contentreferences referenced by containers. This will break if there is a recursive extends or includes in the template code. Due to the use of an empty Context, any extends or include tags with dynamic arguments probably won't work.
		
		"""
        template = DjangoTemplate(self.code)

        # Build a tree of the templates we're using, placing the root template first.
        levels = build_extension_tree(template.nodelist)

        contentlet_specs = []
        contentreference_specs = SortedDict()
        blocks = {}

        for level in reversed(levels):
            level.initialize()
            contentlet_specs.extend(itertools.ifilter(lambda x: x not in contentlet_specs, level.contentlet_specs))
            contentreference_specs.update(level.contentreference_specs)
            for name, block in level.blocks.items():
                if block.block_super:
                    blocks.setdefault(name, []).append(block)
                else:
                    blocks[name] = [block]

        for block_list in blocks.values():
            for block in block_list:
                block.initialize()
                contentlet_specs.extend(itertools.ifilter(lambda x: x not in contentlet_specs, block.contentlet_specs))
                contentreference_specs.update(block.contentreference_specs)

        return contentlet_specs, contentreference_specs
Ejemplo n.º 15
0
    def get_min_expected_inventory_in_interval(
        self, product_id, start_time, end_time, inventory_status_id=None, scheduled_adjustment_exclude_id=None
    ):
        starting_inventory = self.get_net_inventory(product_id, inventory_status_id)
        sa_qs = self.scheduled_adjustments.by_product(product_id).before(end_time).pending_only()
        if scheduled_adjustment_exclude_id:
            sa_qs.exclude(id=scheduled_adjustment_exclude_id)

        scheduled_adjustments = list(sa_qs)

        scheduled_adjustments_before_interval = ifilter(lambda sa: sa.expected_time < start_time, scheduled_adjustments)
        scheduled_adjustments_during_interval = ifilter(
            lambda sa: sa.expected_time >= start_time, scheduled_adjustments
        )

        net_sa_before_interval = sum(sa.qty for sa in scheduled_adjustments_before_interval)

        current_inventory = starting_inventory + net_sa_before_interval
        min_inventory = current_inventory

        for sa in scheduled_adjustments_during_interval:
            current_inventory += sa.qty
            min_inventory = min(current_inventory, min_inventory)

        return min_inventory
Ejemplo n.º 16
0
def merge_schemas(to_schema, from_schema, inplace=False):
    """Recursively merge from_schema into to_schema

    Takes care of leaving to_schema intact if inplace is False (default).
    Returns a new Schema instance if inplace is False or to_schema is a Schema
    class not an instance or the changed to_schema.

    """
    # Nested schemas may be Schema (sub-)classes instead of instances.
    # Copying Schema classes does not work, so we just create an instance.
    if isclass(to_schema) and issubclass(to_schema, validators.Schema):
        to_schema = to_schema()
    elif not inplace:
        to_schema = copy_schema(to_schema)

    # Recursively merge child schemas
    is_schema = lambda f: isinstance(f[1], validators.Schema)
    seen = set()
    for k, v in ifilter(is_schema, to_schema.fields.iteritems()):
        seen.add(k)
        from_field = from_schema.fields.get(k)
        if from_field:
            v = merge_schemas(v, from_field)
            to_schema.add_field(k, v)

    # Add remaining fields if we can
    can_add = lambda f: f[0] not in seen and can_add_field(to_schema, f[0])
    for field in ifilter(can_add, from_schema.fields.iteritems()):
        to_schema.add_field(*field)

    return to_schema
Ejemplo n.º 17
0
    def iter_nodes_local_first(self, ring, partition):
        """
        Yields nodes for a ring partition.

        If the 'write_affinity' setting is non-empty, then this will yield N
        local nodes (as defined by the write_affinity setting) first, then the
        rest of the nodes as normal. It is a re-ordering of the nodes such
        that the local ones come first; no node is omitted. The effect is
        that the request will be serviced by local object servers first, but
        nonlocal ones will be employed if not enough local ones are available.

        :param ring: ring to get nodes from
        :param partition: ring partition to yield nodes for
        """

        primary_nodes = ring.get_part_nodes(partition)
        num_locals = self.app.write_affinity_node_count(len(primary_nodes))
        is_local = self.app.write_affinity_is_local_fn

        if is_local is None:
            return self.app.iter_nodes(ring, partition)

        all_nodes = itertools.chain(primary_nodes, ring.get_more_nodes(partition))
        first_n_local_nodes = list(itertools.islice(itertools.ifilter(is_local, all_nodes), num_locals))

        # refresh it; it moved when we computed first_n_local_nodes
        all_nodes = itertools.chain(primary_nodes, ring.get_more_nodes(partition))
        local_first_node_iter = itertools.chain(
            first_n_local_nodes, itertools.ifilter(lambda node: node not in first_n_local_nodes, all_nodes)
        )

        return self.app.iter_nodes(ring, partition, node_iter=local_first_node_iter)
Ejemplo n.º 18
0
def l_of_d_intersection(ld0, ld1, keys):
    """ Find intersection between a list of dicts and a list of dicts/objects

    :param ld0 :type [DictType] or :type [AnyObject]
    :param ld1 :type [DictType]
    :param keys :type ListType

    :returns intersection of ld0 and ld1 where key is equal.
                 At best, will return `ld0` in full. At worst: [].
    """
    list0, list1 = ld0, ld1
    if len(list0) and type(list0[0]) is not DictType:
        _class = type(list0[0])
        list0 = map(obj_to_d, list0)
    if len(list1) and type(list1[0]) is not DictType:
        _class = type(list1[0])
        list1 = map(obj_to_d, list1)

    processed_ld0 = frozenset(
        ifilter(None, imap(lambda (idx, obj): normalise(idx, obj, keys, id(obj)),
                           enumerate(list0))))

    processed_ld1 = frozenset(
        ifilter(None, imap(lambda (idx, obj): normalise(idx, obj, keys, id(obj)),
                           enumerate(list1))))

    return (ld0[res.idx]
            for result in processed_ld0.intersection(processed_ld1)
            for res in result.values())
Ejemplo n.º 19
0
def articles_from_issue(path, issue_filename):
    issue = graph.resource(URIRef(u'http://miskinhill.com.au/' + path))
    for article in ifilter(lambda r: has_type(r, MHS.Article), issue.subjects(DC.isPartOf)):
        assert article.identifier.startswith(issue.identifier), article.identifier
        generate_pdf(os.path.join(path, issue_filename),
                os.path.join(path, article.identifier.rsplit('/', 1)[1] + '.pdf'),
                article.value(MHS.startPage).toPython() + issue.value(MHS.frontMatterExtent).toPython(),
                article.value(MHS.endPage).toPython() + issue.value(MHS.frontMatterExtent).toPython(),
                entities(striptags(article.value(DC.title).toPython())),
                entities('; '.join(c.value(FOAF.name).toPython() for c in article.objects(DC.creator))))
    for review in ifilter(lambda r: has_type(r, MHS.Review), issue.subjects(DC.isPartOf)):
        assert review.identifier.startswith(issue.identifier + 'reviews/'), review.identifier
        generate_pdf(os.path.join(path, issue_filename),
                os.path.join(path, 'reviews', review.identifier.rsplit('/', 1)[1] + '.pdf'),
                review.value(MHS.startPage).toPython() + issue.value(MHS.frontMatterExtent).toPython(),
                review.value(MHS.endPage).toPython() + issue.value(MHS.frontMatterExtent).toPython(),
                entities('Review of ' + ' and '.join(striptags(b.value(DC.title).toPython()) for b in review.objects(MHS.reviews))),
                entities('; '.join(c.value(FOAF.name).toPython() for c in review.objects(DC.creator))))
    for obituary in ifilter(lambda r: has_type(r, MHS.Obituary), issue.subjects(DC.isPartOf)):
        assert obituary.identifier.startswith(issue.identifier), obituary.identifier
        if obituary.value(DC.title):
            title = striptags(obituary.value(DC.title).toPython())
        else:
            title = 'In memoriam %s' % obituary.value(MHS.obituaryOf).value(FOAF.name).toPython()
        generate_pdf(os.path.join(path, issue_filename),
                os.path.join(path, obituary.identifier.rsplit('/', 1)[1] + '.pdf'),
                obituary.value(MHS.startPageInFrontMatter).toPython(),
                obituary.value(MHS.endPageInFrontMatter).toPython(),
                entities(title),
                entities('; '.join(c.value(FOAF.name).toPython() for c in obituary.objects(DC.creator))))
Ejemplo n.º 20
0
def main():
    """
    create or append a logfile with name yyyy-mm.log as an excerpt of mediatum.log
    of lines beginning with period and containing the string 'INFO' and containing one of the strings:
    'GET', 'POST' or 'HEAD are excerpted
    usage: find /home/congkhacdung/logrotated/ -type f -iname 'mediatum.*.log' | sort | xargs cat | python bin/stats.py --skip-ip 127.0.0.1 --skip-ip 129.187.87.37 2018 2
    """
    parser = argparse.ArgumentParser(description='Extract info needed for statistics.')
    parser.add_argument('--skip-ip', dest='skip_ip', action='append', default=[], help='ip to skip')
    parser.add_argument('year', type=int, help='year')
    parser.add_argument('month', type=int, help='month')
    args = parser.parse_args()
    period = "{:4}-{:0>2}".format(args.year, args.month)
    skip_ip = args.skip_ip

    outdir = os.path.join(config.get("logging.save", config.get("logging.path", "/tmp")))
    match = re.compile('^({period}.{{17}}).*(INFO).{{2}}(.*(?:GET|POST|HEAD).*)'.format(period=period)).match
    lines = sys.stdin
    lines = imap(match, lines)
    lines = ifilter(None, lines)
    lines = imap(operator.methodcaller('groups'), lines)

    skip_ip_pattern = map("([^0-9.]{}[^0-9.])".format, skip_ip)
    skip_ip_pattern = '|'.join(skip_ip_pattern)
    match = re.compile(skip_ip_pattern).match
    lines = ifilter(lambda g: not match(g[2]), lines)
    lines = imap(operator.concat, lines, repeat(("\n",)))
    lines = imap("".join,lines)

    with tempfile.NamedTemporaryFile(dir=outdir) as tmpfile:
        tmpfile.writelines(lines)
        tmpfile.flush()
        init.full_init()
        buildStatAll([], period, tmpfile.name)
Ejemplo n.º 21
0
 def __init__(self, cfdUnitID, unitMgrID = 0, cmdrRating = 0, peripheryID = 0, unit = None, **kwargs):
     super(UnitsListItem, self).__init__()
     creatorFullName = ''
     vehiclesNames = tuple()
     playersCount = 0
     commandSize = 0
     state = 0
     isRosterSet = False
     if unit:
         creatorDBID, creator = next(itertools.ifilter(lambda (dbID, p): p['role'] & UNIT_ROLE.COMMANDER_UPDATES > 0, unit._players.iteritems()), (None, None))
         if creator is not None:
             creatorFullName = g_lobbyContext.getPlayerFullName(creator['nickName'], clanAbbrev=creator.get('clanAbbrev'), pDBID=creatorDBID)
         freeSlots = unit.getFreeSlots()
         playersSlots = unit.getPlayerSlots()
         state = unit.getState()
         vehicles = g_itemsCache.items.getVehicles(REQ_CRITERIA.INVENTORY)
         matches = unit.getRoster().matchVehicleListToSlotList(vehicles.keys(), freeSlots)
         vehiclesNames = tuple(itertools.imap(lambda x: vehicles[x].shortUserName, set(matches.keys())))
         playersCount = len(playersSlots)
         commandSize = len(playersSlots) + len(freeSlots)
         isRosterSet = unit.isRosterSet(ignored=CREATOR_ROSTER_SLOT_INDEXES)
     self.cfdUnitID = cfdUnitID
     self.unitMgrID = unitMgrID
     self.creator = creatorFullName
     self.rating = cmdrRating
     self.peripheryID = peripheryID
     self.playersCount = playersCount
     self.commandSize = commandSize
     self.vehicles = vehiclesNames
     self.state = UnitState(state)
     self.isRosterSet = isRosterSet
     return
Ejemplo n.º 22
0
    def add(self, *urls):
        if not urls:
            return

        def readfile(url):
            with closing(urlopen(url) if url.startswith(('http://', 'https://')) else open(url, 'r')) as f:
                return f.read().splitlines()

        clean_playlist = lambda lines: ifilter(None, imap(str.strip, imap(str, lines)))
        parse_null = lambda url: (url,)

        parsers = {
                '.m3u': lambda url: ifilter(lambda line: not line.startswith('#'), readfile(url)),
                '.pls': lambda url: imap(lambda line: line.split('=')[1], ifilter(lambda line: line.startswith('File'), readfile(url))),
                }

        urls = flatten(imap(lambda url: clean_playlist(parsers.get(os.path.splitext(url)[1], parse_null)(url)), urls))

        cmds = imap(lambda url: ('addid', url), urls)

        try:
            self._mpd.bulk_do(cmds)

        except CommandError:
            self.context.notify('error', _('Songs not found'))

        else:
            self.refresh()
Ejemplo n.º 23
0
def compute_possibles(letters, slots, dictionary_words, context):
    """
    computes possible solution words from given
    dictionary, optionally available slots

    words - iterator of all words in the dictionary
    letters - sorted list of letters on board
    slots - number of available slots

    returns - iterator of possible solution words
    """

    words = dictionary_words

    # if we have a known number of slots filter
    # our word list down to words w/ that manny letters
    if slots:
        words = ifilter(f.word_len(slots), words)

    # filter our word list down to words who's
    # letters are a subset of the given letters
    words = ifilter(f.letter_subset(letters), words)

    # we now have our final iterator of possible solutions
    return words
Ejemplo n.º 24
0
def territory_children_codes(territory_code, include_self=False):
    """ Return a set of subdivision codes from all sub-levels.

    All returned codes are normalized, including self.
    """
    codes = set()

    code = normalize_territory_code(territory_code)

    # We have a country code, look for matching subdivisions in one pass.
    if code in supported_country_codes():
        codes.update(imap(
            attrgetter('code'),
            ifilter(lambda subdiv: subdiv.country_code == code, subdivisions)))

    # Engage the stupid per-level recursive brute-force search as pycountry
    # only expose the child-parent relationship upwards.
    else:
        direct_children_codes = set(imap(
            attrgetter('code'),
            ifilter(lambda subdiv: subdiv.parent_code == code, subdivisions)))
        for child_code in direct_children_codes:
            codes.update(
                territory_children_codes(child_code, include_self=True))

    if include_self:
        codes.add(code)

    return codes
Ejemplo n.º 25
0
    def processImages(self, folder, params):
        """
        Process the images of a directory. This includes computing descriptors
        as well as training ITQ and computing hash codes.

        :param folder: A folder to process images on.
        """
        def oldestFileId(item):
            """
            Find the oldest file in an item, and return its id.

            :param item: An item document, or minimally a dictionary with the item id.
            :returns: The id of the oldest file, or False if the item has no files.
            """
            files = ModelImporter.model('item').childFiles(item,
                                                           limit=1,
                                                           sort=[('created', SortDir.ASCENDING)])
            try:
                return (str(item['_id']), files[0]['_id'])
            except Exception:
                return (False, False)

        # TODO Filter items by supported mime types for SMQTK
        items = itertools.ifilter(lambda item: 'smqtk_uuid' not in item.get('meta', {}),
                                  ModelImporter.model('folder').childItems(folder))

        self._processImages(folder, itertools.ifilter(None,
                                                      itertools.imap(oldestFileId, items)))
Ejemplo n.º 26
0
def truncatable_primes():

    one_digit_primes = set(['2','3','5','7'])
    forbidden_numbers = set(['4','6','8'])

    def is_candidate(number):
        num_str = str(number)

        if len(num_str) < 2: return False

        if not set([num_str[0], num_str[-1]]) < one_digit_primes:
            return False

        digits = set([ digit for digit in num_str ])

        if digits & forbidden_numbers:
            return False

        if '2' in digits and num_str[0] != '2':
            return False

        if '5' in digits and num_str[0] != '5':
            return False

        return True

    trunc_primes = []

    for prime in ifilter(is_truncatable,ifilter(is_candidate,primes)):
        trunc_primes.append(prime)
        if len(trunc_primes) == 11: break

    return trunc_primes
Ejemplo n.º 27
0
def merge_reference_graph(reference_graph):
	# create a copy 
	merged_graph = nx.DiGraph(reference_graph)
	# singletons ? 
	out_d = merged_graph.out_degree()
	in_d = merged_graph.in_degree()
	singletons = [x for x, deg in in_d.items() if (deg == 1) and out_d[x] == 1]
	# bunch them
	induced = merged_graph.subgraph(singletons).to_undirected()
	buckets = itertools.ifilter(lambda x: len(x) > 1, nx.networkx.connected_components(induced.to_undirected()))
	is_entry_point = lambda x: reference_graph.pred[x].items()[0][0] not in singletons
	is_exit_point = lambda x: reference_graph.succ[x].items()[0][0] not in singletons
	n_buckets = 0
	# Compact them in the merged graph
	for bunch in buckets:
		entry_point = reference_graph.predecessors_iter(
			itertools.ifilter(is_entry_point, bunch).next()).next()  # we know there's exactly 1 predecessor and thus 1 entry_point whose pred is in the graph
		exit_point = reference_graph.successors_iter(
			itertools.ifilter(is_exit_point, bunch).next()).next()  # we know there's exactly 1 successor and thus 1 exit point  whose succ exists in the graph
		# print "Compacting chain (unordered): %s"%" / ".join(bunch)
		meta_node_lbl = "_".join(bunch)
		# color the meta node with the 'ref_list' attribut from the first node of the meta node
		ref_list_meta_node = []
		fonction_ref = lambda x: merged_graph.node[x]['ref_list'].keys()
		for node in bunch:
			ref_list_meta_node += fonction_ref(node)
		merged_graph.add_node(meta_node_lbl, length=len(bunch), ref_list=set(ref_list_meta_node))  # je donne un poids au noeud
		merged_graph.remove_nodes_from(bunch)  # j'efface tout
		merged_graph.add_edge(entry_point, meta_node_lbl)  # j'ajoute une arrete entre mon noeud précédent le méta noeud et le méta noeud
		merged_graph.add_edge(meta_node_lbl, exit_point)  # j'ajoute une arrete entre mon noeuds suivant le méta noeud le méta noeud
		n_buckets += 1
	logger.info("Found and compacted %d linear chains", n_buckets)
	logger.info("Reducing graph size from %d to %d", len(reference_graph), len(merged_graph))
	return (merged_graph)
Ejemplo n.º 28
0
 def getSubDevicesGen(self, devfilter=None):
     """ get all the devices under and instance of a DeviceGroup """
     devices = ifilter(lambda dev:self.checkRemotePerm(ZEN_VIEW, dev), self._getSubdevices())
     if devfilter:
         devices = ifilter(devfilter, devices)
     for device in devices:
         yield device
Ejemplo n.º 29
0
def build_graph_from_feature_tuples(X, tol, input_graph=AG):
    G = nx.Graph()
    # the list is ordered by layer and input_id
    nb_layers = X[-1][0][0] + 1
    input_node_count = input_graph.number_of_nodes()
    # Create edges, and crate nodes if needed
    for i in xrange(nb_layers - 1):
        # for each current node
        for c in itertools.ifilter(lambda x: x[0][0] == i
                                   and (x[1] > tol or x[1] < -tol), X):
            input_id = c[0][1]
            src = input_id + (i * input_node_count)
            for n in itertools.ifilter(lambda x: x[0][0] == i + 1
                                       and (x[1] > tol or x[1] < -tol), X):

                tgt_in_id = n[0][1]
                tgt = tgt_in_id + ((i + 1) * input_node_count)
                # Check that input_id are real adajcent in the global
                # adjacency matrix. Beware, ids in adj matrix starts at 0
                if input_graph.has_edge(input_id - 1, tgt_in_id - 1) or input_id == tgt_in_id:

                    if src not in G:
                        data = create_node_data(i, input_id, c[1])
                        G.add_node(src, data)

                    if tgt not in G:
                        data = create_node_data(i + 1, tgt_in_id, n[1])
                        G.add_node(tgt, data)

                    G.add_edge(src, tgt)

    return G
Ejemplo n.º 30
0
 def compute_upto(self, n):
     if n > self.__upto:
         new_possibles = (i for i in xrange(self.__upto+1, n+1) if i % 2 != 0)
         # first, loop over all of our existing primes, and remove any definite
         # non-primes from the possibles
         for cmp_val in ifilter(lambda x: x*x <= n, self.primeset):
             # see comment below about partial() and cmp_val
             new_possibles = ifilter(partial(not_divisible_by,cmp_val), new_possibles)
         
         # at this point we know the first thing in new_possibles is prime (or
         # there isn't anything in new possibles)
         cmp_val = next_or_none(new_possibles)
         while cmp_val is not None and cmp_val**2 <= n:
             #print "Appending: %d" % cmp_val
             self.primeset.append(cmp_val)
 
             # just using cmp_val directly in a lambda causes weird behavior, 
             # because the filter is applied lazily, so the value of cmp_val has changed
             # by the time it's actually used. using partial evalutes cmp_val to create
             # a new function. kind of weird, I've never seen this happen in a language 
             # other than javascript before
             new_possibles = ifilter(partial(not_divisible_by, cmp_val), new_possibles)
             #print "New Possibles: %s" % str(new_possibles)
             cmp_val = next_or_none(new_possibles)
         
         self.primeset.extend(new_possibles)
         
         # if we get here, we should have all the primes up to n
         self.__upto = n
Ejemplo n.º 31
0
def partition(pred, iterable):
    "Use a predicate to partition entries into false entries and true entries"
    # partition(is_odd, range(10)) --> 0 2 4 6 8   and  1 3 5 7 9
    t1, t2 = itertools.tee(iterable)
    return itertools.ifilterfalse(pred, t1), itertools.ifilter(pred, t2)
Ejemplo n.º 32
0
def open_filter_load(pattern='*.root', filter_keyfunc=None):
    wrps = dir_content(pattern)
    wrps = itertools.ifilter(filter_keyfunc, wrps)
    wrps = load(wrps)
    return wrps
Ejemplo n.º 33
0
        elif x.count('out of memory'):
            writeline = False
            printLog('ERROR: Omitting line from table - problem is too large')

    #speedStr = 'GB/s'
    #if args.routine == 'stablesort':
    #    speedStr = 'MKeys/s'
    #if args.routine == 'sort':
    #    speedStr = 'MKeys/s'

    #speedStr = 'GB/s'
        speedStr = 'MKeys/s'
    if writeline:
        try:
            output = itertools.ifilter(lambda x: x.count(speedStr), output)
            output = list(itertools.islice(output, None))
            thisResult = re.search('\d+\.*\d*e*-*\d*$', output[-1])
            thisResult = float(thisResult.group(0))
            thisResult = (params.x, params.device, params.precision,
                          params.label, thisResult)

            outputRow = ''
            for x in thisResult:
                outputRow = outputRow + str(x) + ','
            outputRow = outputRow.rstrip(',')
            table.write(outputRow + '\n')
            table.flush()
        except:
            printLog('ERROR: Exception occurs in GFLOP parsing')
    else:
 def count(self, value):
     return sum(
         [1 for _ in itertools.ifilter(lambda v: v == value, iter(self))])
Ejemplo n.º 35
0
            exclude_unit_path = args.exclude_unit_dir + "_".join(
                coprus_and_lang) + ".tsv"

            sys.stderr.write('reading ignore unit ids: %s\n' %
                             exclude_unit_path)
            exclude_unit_ids = get_unit_id_set(exclude_unit_path)
        else:
            exclude_unit_ids = set()

        annotations += data.load_unambiguous_annotations(
            ssc_file, exclude_unit_ids)

    # Filtering out annotations that have multiple groups
    sys.stderr.write('filtering out multi-group annotations\n')
    annotations = list(
        itertools.ifilter(lambda x: isinstance(x.get_group_number(), int),
                          annotations))

    classifier = models.OptionAwareNaiveBayesLeftRightCutoff(window_size=5,
                                                             cutoff=9)
    sys.stderr.write('training classifier\n')
    classifier.train(annotations)

    sys.stderr.write('processing probabilities\n')

    # If exclude folder is given, find a list of exclude units for the ANNOTATED
    # file and only leave those
    if args.exclude_unit_dir:
        coprus_and_lang = get_corpus_and_language(args.annotate)
        exclude_unit_path = args.exclude_unit_dir + "_".join(
            coprus_and_lang) + ".tsv"
Ejemplo n.º 36
0
def region_mask(cube, region_name):

    # mask cube to country
    import cartopy.io.shapereader as shpreader
    import itertools
    from iris.analysis.geometry import geometry_area_weights
    import numpy.ma as ma

    ### Guess bounds if currently not specified
    if cube.coord('latitude').bounds == None:
        cube.coord('latitude').guess_bounds()
    if cube.coord('longitude').bounds == None:
        cube.coord('longitude').guess_bounds()

    # get countries (resolution = 10m, 50m, 110m )
    shpfilename = shpreader.natural_earth(category='cultural',
                                          name='admin_0_countries',
                                          resolution='110m')
    reader = shpreader.Reader(shpfilename)

    # list available attributes
    all_countries = reader.records()
    country = next(all_countries)
    # print(country.attributes.keys())

    # get all values of an attribute
    key = 'name_long'
    values = set()
    all_countries = reader.records()
    for country in all_countries:
        values.add(country.attributes[key])
    # print( key+': '+ ', '.join(values) )

    # extract countries matching criteria - is there an easier way???
    country_crit = lambda country: country.attributes[
        'name_long'] == region_name  ## e.g., 'China'
    # country_crit = lambda country: country.attributes['continent'] == 'Asia'
    # country_crit = lambda country: country.attributes['region_un'] == 'Asia'
    # country_crit = lambda country: country.attributes['subregion'] == 'Eastern Asia'

    all_countries = reader.records()
    countries = itertools.ifilter(country_crit, all_countries)

    # work out area weights of single field's intersection with selected countries
    # !!! need to make generic (get first field from cube)

    country = next(countries)
    print('Getting field intersection area with ' +
          country.attributes['name_long'])
    area_weights = geometry_area_weights(cube, country.geometry)

    for country in countries:
        print('Getting field intersection area with ' +
              country.attributes['name_long'])
        area_weights += geometry_area_weights(cube, country.geometry)

    # create a mask from the area weights
    mask = np.where(area_weights > 0, False, True)

    masked_cube = cube.copy()

    # NB: this combines the mask and the data's existing mask as required
    masked_cube.data = ma.array(masked_cube.data, mask=mask)

    return masked_cube
Ejemplo n.º 37
0
def main():

    if len(sys.argv) < 3:
        sys.exit("use: %s gff variants" % sys.argv[0])

    # load all the annotations, because pybedtools is broken..
    annotations = pybedtools.BedTool(sys.argv[1])
    annotations._isbam = False
    an_tabix = pysam.Tabixfile(sys.argv[1])

    variants = vcf.Reader(filename=sys.argv[2])

    print_header()

    # for each mrna pick all exons with Target attribute
    # and all variants in the mrna
    # for each variant, if in exon, output with exact coords,
    # otherwise output with NA and mrna block

    def has_target(i):
        return 'Target' in i.attrs

    def exon_target(i):
        target = i.attrs['Target']
        chrom, start, end, _ = target.split(" ", 4)
        return pybedtools.Interval(chrom, int(start), int(end))

    def is_type(i, itype):
        return i.fields[2] == itype

    def is_source(i, src):
        return i.fields[1] == src

    for mrna in itertools.ifilter(lambda x: is_type(x, 'mRNA'), annotations):
        feats = fetch_intervals(an_tabix, mrna)
        exons0 = filter(has_target, filter(lambda x: is_type(x, 'exon'),
                                           feats))

        # pick only the exons mapping to the putative chromosome
        exons = filter(lambda x: exon_target(x).chrom == mrna.chrom, exons0)

        # no exons means no chromosome information, we don't need such variants
        if not exons:
            continue

        # use features from mvz pipeline
        mvz = filter(lambda x: is_source(x, 'mvz-annot'), feats)

        # pick the exon range
        exon_min = min(exon_target(x).start for x in exons)
        exon_max = max(exon_target(x).end for x in exons)

        mvars = list(variants.fetch(mrna.chrom, mrna.start, mrna.end))

        for var in mvars:
            f = list(pybed_find_features(exons, var.CHROM, var.POS))

            # if found a mapped exon, translate the coordinates
            # relative to the exon start
            if f:
                # pick the lowest exon
                minex = f[0]
                refpos = "%d" % (exon_target(minex).start + var.POS -
                                 minex.start)
            else:
                refpos = "NA"

            # list of feature types from mvz annotation pipeline (CDS, 3utr, 5utr..)
            mvzf = list(pybed_find_features(mvz, var.CHROM, var.POS))
            if mvzf:
                mvz_type = ",".join(i.fields[2] for i in mvzf)
            else:
                mvz_type = "NA"

            # try to pick mvz CDS feature and extract the Name(s) from it
            mvz_cds = filter(lambda x: is_type(x, 'CDS'), mvzf)
            if mvz_cds:
                try:
                    mvz_name = ",".join(i.attrs['Name'] for i in mvz_cds
                                        if 'Name' in i.attrs)
                except:
                    print >> sys.stderr, "error at var:", var.CHROM, var.POS
            else:
                mvz_name = "NA"

            print "\t".join(
                map(str, [
                    var.CHROM,
                    var.POS,
                    var.QUAL,
                    var.FILTER,
                    var.var_type,
                    var.var_subtype,
                    var.INFO['DP'],
                    "\t".join(str(x) for x in var.INFO['DP4']),
                    mrna.name,
                    mvz_name,
                    mrna.end - mrna.start,
                    exon_min,
                    exon_max,
                    refpos,
                    mvz_type,
                ]))
Ejemplo n.º 38
0
 def slaves(self, fltr=""):
     return list(
         map(
             lambda x: slave.MesosSlave(x),
             itertools.ifilter(lambda x: fltr in x["id"],
                               self.state["slaves"])))
Ejemplo n.º 39
0
def mindmap_to_blog(mindmap, do_cleanup=False):
    #===========================================================
    # cleanup previous markdown files
    #===========================================================
    # {{{
    if do_cleanup:
        print '\n## cleanup old markdown files created by freeplane.py blog mode'
        dpath = os.path.dirname(mindmap.fpath)
        for root, ls_dname, ls_fname in os.walk(dpath):
            for fname in ls_fname:
                if (fname.endswith('.md')):
                    # is Markdown file
                    fpath = os.path.join(root, fname)
                    with open(fpath, 'r') as f:
                        f.seek(-2 * len(blog_mode_footnote), os.SEEK_END)
                        last = f.read().strip()
                    if last.endswith(blog_mode_footnote):
                        # is generated, so remove the markdown file
                        print '### remove file ' + fpath
                        os.remove(fpath)

    # }}}

    #===========================================================
    # find blog mode settings
    #===========================================================
    # {{{

    print '\n## analyze blog mode settings'
    setting_root = next(itertools.ifilter(lambda x: x.core.lower() == 'blog mode settings', mindmap.root.ls_sub), None)
    assert (setting_root != None), 'Error: cannot find "blog mode settings" node'

    #-------------------------------------------------------
    # node type & link type

    node_type = dict()
    link_type = dict()
    for node in itertools.islice(iter(setting_root), 1, None):
        type_name = node.core.lower()

        if (type_name == 'special nodes'):
            continue

        if (type_name == 'link file types'):
            for (keyword, value) in node.attribute.items():
                link_type[keyword] = value.split()
            continue

        try:
            node_type[type_name].append(node)
        except KeyError:
            node_type[type_name] = [node]

    print '### node type'
    for (type_name, ls_node) in node_type.items():
        s = '%s = ' % type_name
        for node in ls_node:
            s += node.format + '|'
        print s.strip('|')

    print '### link type'
    for (type_name, ls_pattern) in link_type.items():
        s = '%s = ' % type_name
        for pattern in ls_pattern:
            s += pattern + '|'
        print s.strip('|')

    #-------------------------------------------------------
    # article's default meta-data

    article_setting = next(itertools.ifilter(lambda n: n.core.lower() == 'article', setting_root), None)
    assert article_setting != None, 'Error: cannot find "article" setting node'
    default_metadata = dict()
    for (key, value) in article_setting.attribute.items():
        default_metadata[key.lower()] = value

    print '### default meta-data'
    for (key, value) in default_metadata.items():
        print '%s = %s' % (key, value)
    # }}}

    #===========================================================
    # set node type according to their format matching with blog mode settings
    #===========================================================
    # {{{

    print '\n## match node format'

    # node_type
    for node in iter(mindmap.root):
        node.type = ''
        for (type_name, ls_node_setting) in node_type.items():
            for node_setting in ls_node_setting:
                if (node.format == node_setting.format):
                    node.type = type_name
                    break
            if len(node.type) > 0:
                break
        if (node.type == ''):
            if node.core.startswith('- '):
                node.type = 'list'
                if (node.parent.type != 'list'):
                    node.list_level = 1
                else:
                    node.list_level = node.parent.list_level + 1

    for node in iter(setting_root):
        node.type = ''
    mindmap.root.type = ''

    # link_type
    for node in itertools.ifilter(lambda n: len(n.link) > 0, mindmap.root):
        node.link_type = ''
        for (type_name, ls_link_type_pattern) in link_type.items():
            for link_type_pattern in ls_link_type_pattern:
                if (fnmatch.fnmatch(node.core, link_type_pattern)):
                    node.link_type = type_name
                    break
            if len(node.link_type) > 0:
                break
    # }}}

    #===========================================================
    # analyze article
    #===========================================================
    # {{{

    print '\n## analyze blog articles'

    for node in itertools.ifilter(lambda n: n.type == 'article', mindmap.root):
        print '\n### analyze article:\n' + node.core.encode('utf-8')

        #-------------------------------------------------------
        # article file path
        dpath = os.path.dirname(mindmap.fpath)
        if node.parent.type == 'directory':
            directory = node.parent.core.replace(' ', '-').replace(':', '..')
            dpath = os.path.join(dpath, directory)

        #-------------------------------------------------------
        # article meta-data
        metadata = dict()
        for (key, value) in node.attribute.items():
            metadata[key.lower()] = value
        metadata['title'] = node.core
        metadata['category'] = directory.replace(' ', '-')
        if ('slug' not in metadata.keys()):
            metadata['slug'] = metadata['title'].lower().replace(' ', '-')

        # set default meta-data
        for (keyword, value) in default_metadata.items():
            if keyword not in metadata.keys():
                metadata[keyword] = value

        if ('modified' not in metadata.keys()) or (metadata['modified'] == ''):
            if ('created' in metadata.keys()):
                metadata['modified'] = metadata['created']
        if ('created' in metadata.keys()):
            metadata['date'] = metadata['created']

        fname = metadata['slug'] + '.md'
        fpath = os.path.join(dpath, fname)

        #-------------------------------------------------------
        # write article
        if not os.path.isdir(dpath):
            os.mkdir(dpath)

        # title
        content = 'title: ' + metadata['title'] + '\n'

        # meta-data
        for (key, value) in metadata.items():
            if (key == 'title'):
                continue
            content += '%s: %s\n' % (key, value)
        content += '\n'

        #-------------------------------------------------------
        # sub node content {{{
        for n in itertools.islice(iter(node), 1, None):
            if n.type == 'skip':
                continue

            if n.type == 'comment':
                map(lambda m: setattr(m, 'type', 'comment'), n.ls_sub)
                continue

            # deal with links
            if len(n.link) > 0:
                # solve relative link
                if n.link.startswith('http://'):
                    # http link
                    link = n.link
                else:
                    # local files
                    # copy local files to directory it supposed to be in: "<directory>/<slug>/", and modify the link in original mindmap file
                    exp_link = directory + '/' + metadata['slug'] + '/' + n.core.replace(' ', '_')
                    if (n.link != exp_link):
                        old_fpath = os.path.join(os.path.dirname(mindmap.fpath), n.link).replace(r'%20', r' ')
                        new_fpath = os.path.join(os.path.dirname(mindmap.fpath), exp_link)
                        if (os.path.dirname(n.link) == os.path.dirname(exp_link)):
                            # just filename is different, rename
                            print 'Info: file (%s) is renamed to (%s)' % (old_fpath, new_fpath)
                            os.rename(old_fpath, new_fpath)
                        else:
                            # copy
                            print 'Info: file (%s) is copied to (%s)' % (old_fpath, new_fpath)
                            if not os.path.isdir(os.path.dirname(new_fpath)):
                                os.mkdir(os.path.dirname(new_fpath))
                            shutil.copyfile(old_fpath, new_fpath)

                        # change the link
                        mindmap.is_changed = True
                        before = 'LINK="' + n.link + '"'
                        after = 'LINK="' + exp_link + '"'
                        mindmap.ls_change.append((before, after))

                    link = metadata['slug'] + '/' + n.core.replace(' ', '_')

                if n.link_type == 'image':
                    content += '![%s](%s)\n' % (n.core, link)
                else:
                    content += '[%s](%s)\n' % (n.core, link)

                continue

            # deal with special nodes
            prefix = ''
            suffix = ''

            if n.type == 'section':
                prefix = '# '
            elif n.type == 'subsection':
                prefix = '## '
            elif n.type == 'subsubsection':
                prefix = '### '
            elif n.type == 'paragraph':
                prefix = '#### '
            elif n.type == 'subparagraph':
                prefix = '##### '
            elif n.type == 'code':
                prefix = '~~~\n'
                suffix = '\n~~~'
            elif n.type == 'list':
                prefix = '    ' * (n.list_level - 1)
                suffix = ''

            content += prefix + n.core + suffix + '\n'

            if len(n.note) > 0:
                content += '\n' + n.note + '\n'
        # }}}

        content += blog_mode_footnote

        #-------------------------------------------------------
        # write the markdown file {{{
        f = open(fpath, 'w')
        print >> f, content.encode('utf-8')
        f.close()
        print '### wrote to ' + fpath
        # }}}

    # }}}

    #===========================================================
    # modify original mindmap and backup
    #===========================================================
    # {{{
    if len(mindmap.ls_change) == 0:
        pass
    else:
        print '\n## change mindmap with backup'

        # backup

        prefix = datetime.datetime.now().strftime('%Y%m%d_%H%M%S.')
        bak_fpath = os.path.join(os.path.dirname(mindmap.fpath), 'bak', prefix + os.path.basename(mindmap.fpath))
        if not os.path.isdir(os.path.dirname(bak_fpath)):
            os.mkdir(os.path.dirname(bak_fpath))
        shutil.copyfile(mindmap.fpath, bak_fpath)
        print '### mindmap file backup at %s' % bak_fpath

        # replace mindmap
        with open(mindmap.fpath, 'r') as f:
            ls_lines = f.readlines()

        for (before, after) in mindmap.ls_change:
            print '### (%s) >> (%s)' % (before, after)
            for (i, line) in enumerate(ls_lines):
                ls_lines[i] = line.replace(before, after)

        with open(mindmap.fpath, 'w') as f:
            f.writelines(ls_lines)
    # }}}

    #===========================================================
    # cleanup not used attachment dir
    #===========================================================
    # {{{
    if (do_cleanup):
        print '\n## cleanup not used attachment dir'
        dpath = os.path.dirname(mindmap.fpath)
        for root, ls_dname, ls_fname in os.walk(dpath):
            # skip the first level
            if (root == dpath) or (os.path.basename(root) == 'bak'):
                continue

            st_dname = set(ls_dname)
            st_slug = set(map(lambda x: x[:-3], ls_fname))
            st_not_used = st_dname - st_slug
            for dname in st_not_used:
                dpath = os.path.join(root, dname)
                print '### remove dir ' + dpath
                shutil.rmtree(dpath)
Ejemplo n.º 40
0
def _sieve(stream):
    # just for fun; doesn't work over a few hundred
    val = stream.next()
    yield val
    for x in ifilter(lambda x: x%val != 0, _sieve(stream)):
        yield x
Ejemplo n.º 41
0
 def filter_out_dummy(iterable):
     return itertools.ifilter(lambda x: x != dummy_fill_value,
                              iterable)
Ejemplo n.º 42
0
 def compute(self, split):
     if self.err < 1e-8:
         return itertools.ifilter(self.func, self.prev.iterator(split))
     return self._compute_with_error(split)
Ejemplo n.º 43
0
 def slaves(self, fltr=""):
     return list(
         map(
             lambda x: slave.MesosSlave(self.config, x),
             itertools.ifilter(lambda x: fltr == x['id'],
                               self.state['slaves'])))
Ejemplo n.º 44
0
def ParseResults(ycsb_result_string, data_type='histogram'):
    """Parse YCSB results.

  Example input for histogram datatype:

    YCSB Client 0.1
    Command line: -db com.yahoo.ycsb.db.HBaseClient -P /tmp/pkb/workloada
    [OVERALL], RunTime(ms), 1800413.0
    [OVERALL], Throughput(ops/sec), 2740.503428935472
    [UPDATE], Operations, 2468054
    [UPDATE], AverageLatency(us), 2218.8513395574005
    [UPDATE], MinLatency(us), 554
    [UPDATE], MaxLatency(us), 352634
    [UPDATE], 95thPercentileLatency(ms), 4
    [UPDATE], 99thPercentileLatency(ms), 7
    [UPDATE], Return=0, 2468054
    [UPDATE], 0, 398998
    [UPDATE], 1, 1015682
    [UPDATE], 2, 532078
    ...

  Example input for hdrhistogram datatype:

    YCSB Client 0.12.0
    Command line: -db com.yahoo.ycsb.db.RedisClient -P /opt/pkb/workloadb
    [OVERALL], RunTime(ms), 29770.0
    [OVERALL], Throughput(ops/sec), 33590.86328518643
    [UPDATE], Operations, 49856.0
    [UPDATE], AverageLatency(us), 1478.0115532734276
    [UPDATE], MinLatency(us), 312.0
    [UPDATE], MaxLatency(us), 24623.0
    [UPDATE], 95thPercentileLatency(us), 3501.0
    [UPDATE], 99thPercentileLatency(us), 6747.0
    [UPDATE], Return=OK, 49856
    ...

  Example input for ycsb version after 0.13.0:

    ...
    Command line: -db com.yahoo.ycsb.db.HBaseClient10 ... -load
    YCSB Client 0.14.0

    Loading workload...
    Starting test.
    ...
    [OVERALL], RunTime(ms), 11411
    [OVERALL], Throughput(ops/sec), 8763.473841030585
    [INSERT], Operations, 100000
    [INSERT], AverageLatency(us), 74.92
    [INSERT], MinLatency(us), 5
    [INSERT], MaxLatency(us), 98495
    [INSERT], 95thPercentileLatency(us), 42
    [INSERT], 99thPercentileLatency(us), 1411
    [INSERT], Return=OK, 100000
    ...

  Example input for timeseries datatype:

    ...
    [OVERALL], RunTime(ms), 240007.0
    [OVERALL], Throughput(ops/sec), 10664.605615669543
    ...
    [READ], Operations, 1279253
    [READ], AverageLatency(us), 3002.7057071587874
    [READ], MinLatency(us), 63
    [READ], MaxLatency(us), 93584
    [READ], Return=OK, 1279281
    [READ], 0, 528.6142757498257
    [READ], 500, 360.95347448674966
    [READ], 1000, 667.7379547689283
    [READ], 1500, 731.5389357265888
    [READ], 2000, 778.7992281717318
    ...

  Args:
    ycsb_result_string: str. Text output from YCSB.
    data_type: Either 'histogram' or 'timeseries' or 'hdrhistogram'.
      'histogram' and 'hdrhistogram' datasets are in the same format, with the
      difference being lacking the (millisec, count) histogram component. Hence
      are parsed similarly.

  Returns:
    A dictionary with keys:
      client: containing YCSB version information.
      command_line: Command line executed.
      groups: list of operation group descriptions, each with schema:
        group: group name (e.g., update, insert, overall)
        statistics: dict mapping from statistic name to value
        histogram: list of (ms_lower_bound, count) tuples, e.g.:
          [(0, 530), (19, 1)]
        indicates that 530 ops took between 0ms and 1ms, and 1 took between
        19ms and 20ms. Empty bins are not reported.
  Raises:
    IOError: If the results contained unexpected lines.
  """
    # TODO: YCSB 0.9.0 output client and command line string to stderr, so
    # we need to support it in the future.
    lines = []
    client_string = 'YCSB'
    command_line = 'unknown'
    fp = io.BytesIO(ycsb_result_string)
    result_string = next(fp).strip()

    def IsHeadOfResults(line):
        return line.startswith('[OVERALL]')

    while not IsHeadOfResults(result_string):
        if result_string.startswith('YCSB Client 0.'):
            client_string = result_string
        if result_string.startswith('Command line:'):
            command_line = result_string
        try:
            result_string = next(fp).strip()
        except StopIteration:
            raise IOError(
                'Could not parse YCSB output: {}'.format(ycsb_result_string))

    if result_string.startswith('[OVERALL]'):  # YCSB > 0.7.0.
        lines.append(result_string)
    else:
        # Received unexpected header
        raise IOError('Unexpected header: {0}'.format(client_string))

    # Some databases print additional output to stdout.
    # YCSB results start with [<OPERATION_NAME>];
    # filter to just those lines.
    def LineFilter(line):
        return re.search(r'^\[[A-Z]+\]', line) is not None

    lines = itertools.chain(lines, itertools.ifilter(LineFilter, fp))

    r = csv.reader(lines)

    by_operation = itertools.groupby(r, operator.itemgetter(0))

    result = collections.OrderedDict([('client', client_string),
                                      ('command_line', command_line),
                                      ('groups', collections.OrderedDict())])

    for operation, lines in by_operation:
        operation = operation[1:-1].lower()

        if operation == 'cleanup':
            continue

        op_result = {'group': operation, data_type: [], 'statistics': {}}
        latency_unit = 'ms'
        for _, name, val in lines:
            name = name.strip()
            val = val.strip()
            # Drop ">" from ">1000"
            if name.startswith('>'):
                name = name[1:]
            val = float(val) if '.' in val or 'nan' in val.lower() else int(
                val)
            if name.isdigit():
                if val:
                    if data_type == TIMESERIES and latency_unit == 'us':
                        val /= 1000.0
                    op_result[data_type].append((int(name), val))
            else:
                if '(us)' in name:
                    name = name.replace('(us)', '(ms)')
                    val /= 1000.0
                    latency_unit = 'us'
                op_result['statistics'][name] = val

        result['groups'][operation] = op_result
    return result
Ejemplo n.º 45
0
 def processes(self, workflow=None, tag=None):
     return ifilter(
         lambda p: (p.workflow == workflow or not workflow) and
         (tag in p.tags or not tag), self.running_processes.values())
Ejemplo n.º 46
0
def multi_substitute(word, rules):
    """ Apply multiple regex rules to 'word'
    http://code.activestate.com/recipes/
    576710-multi-regex-single-pass-replace-of-multiple-regexe/
    """
    flags = rules[0]['flags']

    # Create a combined regex from the rules
    tuples = ((p, r['match']) for p, r in enumerate(rules))
    regexes = ('(?P<match_%i>%s)' % (p, r) for p, r in tuples)
    pattern = '|'.join(regexes)
    regex = re.compile(pattern, flags)
    resplit = re.compile('\$(\d+)')

    # For each match, look-up corresponding replace value in dictionary
    rules_in_series = ifilter(itemgetter('series'), rules)
    rules_in_parallel = (r for r in rules if not r['series'])

    try:
        has_parallel = [rules_in_parallel.next()]
    except StopIteration:
        has_parallel = []

    # print('================')
    # pprint(rules)
    # print('word:', word)
    # print('pattern', pattern)
    # print('flags', flags)

    for _ in chain(rules_in_series, has_parallel):
        # print('~~~~~~~~~~~~~~~~')
        # print('new round')
        # print('word:', word)
        # found = list(regex.finditer(word))
        # matchitems = [match.groupdict().items() for match in found]
        # pprint(matchitems)
        prev_name = None
        prev_is_series = None
        i = 0

        for match in regex.finditer(word):
            item = ifilter(itemgetter(1), match.groupdict().iteritems()).next()

            # print('----------------')
            # print('groupdict:', match.groupdict().items())
            # print('item:', item)

            if not item:
                continue

            name = item[0]
            rule = rules[int(name[6:])]
            series = rule.get('series')
            kwargs = {'count': rule['count'], 'series': series}
            is_previous = name is prev_name
            singlematch = kwargs['count'] is 1
            is_series = prev_is_series or kwargs['series']
            isnt_previous = bool(prev_name) and not is_previous

            if (is_previous and singlematch) or (isnt_previous and is_series):
                continue

            prev_name = name
            prev_is_series = series

            if resplit.findall(rule['replace']):
                splits = resplit.split(rule['replace'])
                words = _gen_words(match, splits)
            else:
                splits = rule['replace']
                start = match.start() + i
                end = match.end() + i
                words = [word[:start], splits, word[end:]]
                i += rule['offset']

            # words = list(words)
            word = ''.join(words)

            # print('name:', name)
            # print('prereplace:', rule['replace'])
            # print('splits:', splits)
            # print('resplits:', resplit.findall(rule['replace']))
            # print('groups:', filter(None, match.groups()))
            # print('i:', i)
            # print('words:', words)
            # print('range:', match.start(), '-', match.end())
            # print('replace:', word)

    # print('substitution:', word)
    return word
Ejemplo n.º 47
0
 def selectColumns(self, row):
     outRows = imap(
         lambda i:row[i], 
         ifilter(lambda i: i in self._getTransColumns(), range(len(row))))
     return outRows
Ejemplo n.º 48
0

address_types = {ArrayType, StringType, FunctionType, AddressType}


def size_arrays_as_pointers(ctype, overrides=()):
    return size(
        ctype,
        overrides=dict(chain(
            izip(address_types, repeat(size(void_pointer_type))), getattr(overrides, 'iteritems', lambda: overrides)()
        ))
    )


float_sizes_to_words = dict(izip(imap(word_type_sizes.__getitem__, float_names), float_names))  # float word sizes
float_ctypes = set(ifilter(lambda cls: issubclass(cls, FloatType), rules(numeric_type_size).iterkeys()))  # CFloat types
float_ctypes_word_types = dict(izip(    # Convert CType to its size then convert that size to machine word type name
    float_ctypes, imap(float_sizes_to_words.__getitem__, imap(rules(numeric_type_size).__getitem__, float_ctypes))
))

strictly_unsigned_ctypes = set(
    ifilter(lambda cls: issubclass(cls, StrictlyUnsigned), rules(numeric_type_size).iterkeys())
) - float_ctypes  # just to be safe even though floats are strictly signed!
strictly_signed_ctypes = set(
    ifilter(lambda cls: issubclass(cls, StrictlySigned), rules(numeric_type_size).iterkeys())
) - float_ctypes

integral_ctypes = set(ifilter(lambda cls: issubclass(cls, IntegralType), rules(numeric_type_size).iterkeys()))


unsigned_ctypes_to_words = dict(izip(imap(word_type_sizes.__getitem__, word_names), word_names))
Ejemplo n.º 49
0
        url = 'https://api.import.io/store/data/c1ea6c12-c034-4b99-9252-81a5395c1701/_query?input/webpage/url=http://www.groupon.com/browse/' + city + '?context=local&' + userkeys
        return url

    backup_page_list = []
    deals_list = []
    page_data_list = []

    # with open('Updated_sample_groupon_page_data_test.json', 'w') as jsonfile:
    # json.dump(page_data_list, jsonfile, encoding='utf8')

    for c in cities[21:23]:
        print c
        try:
            trial = [json.load(urlopen(total_page_extractor(c)))['results'] for each in range(0, 10)]

            trial = itertools.ifilter(None, trial).next()
            for page in trial:
                total_pages = int(page['total_page_numb/_text'])
                print "%s total pages for %s" % (total_pages, c)
            # print type(total_pages)
            # print trial
            for s in range(1, total_pages + 1):
                city = c
                page_number = str(s)
                # print page_number
                deal_list_crawler = 'https://api.import.io/store/data/affd236e-be39-4952-9100-d7f3564add10/_query?input/webpage/url=http://www.groupon.com/browse%2F' + city + '%3Fcontext%3Dlocal%26page%3D' + page_number + '&' + userkeys
                result = json.load(urlopen(deal_list_crawler))['results']
                while True:
                    if result != []:
                        for d in result:
                            del d['all_deals_links/_source']
Ejemplo n.º 50
0
def not_none(it):
    return itertools.ifilter(None, it)
Ejemplo n.º 51
0
Archivo: _vc.py Proyecto: yongzhi2/meld
def partition(pred, iterable):
    t1, t2 = itertools.tee(iterable)
    return (list(itertools.ifilterfalse(pred, t1)),
            list(itertools.ifilter(pred, t2)))
Ejemplo n.º 52
0
 def all_tids(self):
     free = self.free_list()
     n = len(self._chunks) * DEFAULT_CHUNK_SIZE
     return ifilter(lambda x: x not in free, imap(self._make_tid,
                                                  xrange(n)))
Ejemplo n.º 53
0
 def iter_blocks(self):
     return ifilter(methodcaller('is_block'), self.get_elements())
Ejemplo n.º 54
0
 def iter_connections(self):
     return ifilter(methodcaller('is_connection'), self.get_elements())
Ejemplo n.º 55
0
def first(condition, seq):
    try:
        return next(ifilter(condition, seq))
    except StopIteration:
        return None
Ejemplo n.º 56
0
        if isValid(sumNs, mcmTot, p): yield s


## ****************** MAIN ******************

limit = 80
cands = range(2, limit + 1)
denoms = [x**2 for x in cands]
mcmTot = mcm(denoms)
print mcmTot
## Global variable, used everywhere. The cases 0 and 1 are manually set.
val = [None, mcmTot] + [mcmTot // d for d in denoms]

## Removing candidates that don't appear in any combination:
sRemove = set([])
for p in ifilter(isPrime, xrange(5, limit + 1)):
    sFound = set(range(p, limit + 1, p))
    for s in possibleP(p):
        sFound.difference_update(p * el for el in s)
    sRemove.update(sFound)
for el in sRemove:
    cands.remove(el)

print "Candidates: %s" % cands
## Now, we divide the candidates in approx. 2 halves:
denoms1 = [x**2 for x in cands[:len(cands) // 2]]
denoms2 = [x**2 for x in cands[len(cands) // 2:]]
mcm_1 = mcm(denoms1)
mcm_2 = mcm(denoms2)
mcmTot = mcm([mcm_1, mcm_2])
mcmTot2 = mcmTot // 2
Ejemplo n.º 57
0
def plotFromDataFile():
    data = []
    """
  read in table(s) from file(s)
  """
    for thisFile in args.datafile:
        if not os.path.isfile(thisFile):
            print 'No file with the name \'{}\' exists. Please indicate another filename.'.format(
                thisFile)
            quit()

        results = open(thisFile, 'r')
        results_contents = results.read()
        results_contents = results_contents.rstrip().split('\n')

        firstRow = results_contents.pop(0)
        print firstRow
        print blas_table_header()
        print firstRow.rstrip() == blas_table_header()
        if firstRow.rstrip() != blas_table_header():
            print 'ERROR: input file \'{}\' does not match expected format.'.format(
                thisFile)
            quit()

        for row in results_contents:
            row = row.split(',')
            row = TableRow(
                BlasTestCombination(row[0], row[1], row[2], row[3], row[4],
                                    row[5], row[6], row[7], row[8], row[9],
                                    row[10], row[11], row[12], row[13],
                                    row[14], row[15], row[16], row[17][1:],
                                    row[17][0], row[18], row[19], row[20]),
                row[21])
            data.append(
                BlasGraphPoint(
                    row.parameters.sizem, row.parameters.sizen,
                    row.parameters.sizek, row.parameters.lda,
                    row.parameters.ldb, row.parameters.ldc,
                    row.parameters.offa, row.parameters.offb,
                    row.parameters.offc, row.parameters.device,
                    row.parameters.order, row.parameters.transa,
                    row.parameters.transb,
                    row.parameters.precision + row.parameters.function,
                    row.parameters.library, row.parameters.label, row.gflops))
    """
  data sanity check
  """
    # if multiple plotvalues have > 1 value among the data rows, the user must specify which to plot
    multiplePlotValues = []
    for option in plotvalues:
        values = []
        for point in data:
            values.append(getattr(point, option))
        multiplePlotValues.append(len(set(values)) > 1)
    if multiplePlotValues.count(True) > 1 and args.plot == None:
        print 'ERROR: more than one parameter of {} has multiple values. Please specify which parameter to plot with --plot'.format(
            plotvalues)
        quit()

    # if args.graphxaxis is not 'problemsize', the user should know that the results might be strange
    #if args.graphxaxis != 'problemsize':
    #  xaxisvalueSet = []
    #  for option in xaxisvalues:
    #    if option != 'problemsize':
    #      values = []
    #      for point in data:
    #        values.append(getattr(point, option))
    #      xaxisvalueSet.append(len(set(values)) > 1)
    #  if xaxisvalueSet.count(True) > 1:
    #    print 'WARNING: more than one parameter of {} is varied. unexpected results may occur. please double check your graphs for accuracy.'.format(xaxisvalues)

    # multiple rows should not have the same input values
    #pointInputs = []
    #for point in data:
    #  pointInputs.append(point.__str__().split(';')[0])
    #if len(set(pointInputs)) != len(data):
    #  print 'ERROR: imported table has duplicate rows with identical input parameters'
    #  quit()
    """
  figure out if we have multiple plots on this graph (and what they should be)
  """
    if args.plot != None:
        multiplePlots = args.plot
    elif multiplePlotValues.count(True) == 1 and plotvalues[
            multiplePlotValues.index(True)] != 'sizek':
        # we don't ever want to default to sizek, because it's probably going to vary for most plots
        # we'll require the user to explicitly request multiple plots on sizek if necessary
        multiplePlots = plotvalues[multiplePlotValues.index(True)]
    else:
        # default to device if none of the options to plot have multiple values
        multiplePlots = 'device'
    """
  assemble data for the graphs
  """
    data.sort(key=lambda row: int(getattr(row, args.graphxaxis)))

    # choose scale for x axis
    if args.xaxisscale == None:
        # user didn't specify. autodetect
        if int(getattr(data[len(data) - 1],
                       args.graphxaxis)) > 2000:  # big numbers on x-axis
            args.xaxisscale = 'log2'
        elif int(getattr(data[len(data) - 1],
                         args.graphxaxis)) > 10000:  # bigger numbers on x-axis
            args.xaxisscale = 'log10'
        else:  # small numbers on x-axis
            args.xaxisscale = 'linear'

    if args.xaxisscale == 'linear':
        plotkwargs = {}
        plottype = 'plot'
    elif args.xaxisscale == 'log2':
        plottype = 'semilogx'
        plotkwargs = {'basex': 2}
    elif args.xaxisscale == 'log10':
        plottype = 'semilogx'
        plotkwargs = {'basex': 10}
    else:
        print 'ERROR: invalid value for x-axis scale'
        quit()

    plots = set(getattr(row, multiplePlots) for row in data)

    class DataForOnePlot:
        def __init__(self, inlabel, inxdata, inydata):
            self.label = inlabel
            self.xdata = inxdata
            self.ydata = inydata

    dataForAllPlots = []
    for plot in plots:
        dataForThisPlot = itertools.ifilter(
            lambda x: getattr(x, multiplePlots) == plot, data)
        dataForThisPlot = list(itertools.islice(dataForThisPlot, None))
        #if args.graphxaxis == 'problemsize':
        #  xdata = [int(row.x) * int(row.y) * int(row.z) * int(row.batchsize) for row in dataForThisPlot]
        #else:
        xdata = [getattr(row, args.graphxaxis) for row in dataForThisPlot]
        ydata = [getattr(row, args.graphyaxis) for row in dataForThisPlot]
        dataForAllPlots.append(DataForOnePlot(plot, xdata, ydata))
    """
  assemble labels for the graph or use the user-specified ones
  """
    if args.graphtitle:
        # use the user selection
        title = args.graphtitle
    else:
        # autogen a lovely title
        title = 'Performance vs. ' + args.graphxaxis.capitalize()

    if args.xaxislabel:
        # use the user selection
        xaxislabel = args.xaxislabel
    else:
        # autogen a lovely x-axis label
        if args.graphxaxis == 'cachesize':
            units = '(bytes)'
        else:
            units = '(datapoints)'

        xaxislabel = args.graphxaxis + ' ' + units

    if args.yaxislabel:
        # use the user selection
        yaxislabel = args.yaxislabel
    else:
        # autogen a lovely y-axis label
        if args.graphyaxis == 'gflops':
            units = 'GFLOPS'
        yaxislabel = 'Performance (' + units + ')'
    """
  display a pretty graph
  """
    colors = ['k', 'y', 'm', 'c', 'r', 'b', 'g']

    for thisPlot in dataForAllPlots:
        getattr(pylab, plottype)(thisPlot.xdata,
                                 thisPlot.ydata,
                                 '{}.-'.format(colors.pop()),
                                 label=thisPlot.label,
                                 **plotkwargs)

    if len(dataForAllPlots) > 1:
        pylab.legend(loc='best')

    pylab.title(title)
    pylab.xlabel(xaxislabel)
    pylab.ylabel(yaxislabel)
    pylab.grid(True)

    if args.outputFilename == None:
        # if no pdf output is requested, spit the graph to the screen . . .
        pylab.show()
    else:
        # . . . otherwise, gimme gimme pdf
        #pdf = PdfPages(args.outputFilename)
        #pdf.savefig()
        #pdf.close()
        pylab.savefig(args.outputFilename, dpi=(1024 / 8))
Ejemplo n.º 58
0
    def _update_parameter_controls(self):
        for control in ifilter(None, self._parameter_controls or []):
            control.set_channel(self._bank_index)

        return
Ejemplo n.º 59
0
def main():
    arg_parser = ArgumentParser(description='Build an XLS performance report.')
    arg_parser.add_argument('sheet_dirs', nargs='+', metavar='DIR', help='directory containing perf test logs')
    arg_parser.add_argument('-o', '--output', metavar='XLS', default='report.xls', help='name of output file')
    arg_parser.add_argument('-c', '--config', metavar='CONF', help='global configuration file')
    arg_parser.add_argument('--include-unmatched', action='store_true',
        help='include results from XML files that were not recognized by configuration matchers')
    arg_parser.add_argument('--show-times-per-pixel', action='store_true',
        help='for tests that have an image size parameter, show per-pixel time, as well as total time')

    args = arg_parser.parse_args()

    logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG)

    if args.config is not None:
        with open(args.config) as global_conf_file:
            global_conf = ast.literal_eval(global_conf_file.read())
    else:
        global_conf = {}

    wb = xlwt.Workbook()

    for sheet_path in args.sheet_dirs:
        try:
            with open(os.path.join(sheet_path, 'sheet.conf')) as sheet_conf_file:
                sheet_conf = ast.literal_eval(sheet_conf_file.read())
        except IOError as ioe:
            if ioe.errno != errno.ENOENT: raise
            sheet_conf = {}
            logging.debug('no sheet.conf for %s', sheet_path)

        sheet_conf = dict(global_conf.items() + sheet_conf.items())

        config_names = sheet_conf.get('configurations', [])
        config_matchers = sheet_conf.get('configuration_matchers', [])

        collector = Collector(make_match_func(config_matchers), args.include_unmatched)

        for root, _, filenames in os.walk(sheet_path):
            logging.info('looking in %s', root)
            for filename in fnmatch.filter(filenames, '*.xml'):
                if os.path.normpath(sheet_path) == os.path.normpath(root):
                  default_conf = None
                else:
                  default_conf = os.path.relpath(root, sheet_path)
                collector.collect_from(os.path.join(root, filename), default_conf)

        config_names.extend(sorted(collector.extra_configurations - set(config_names)))

        sheet = wb.add_sheet(sheet_conf.get('sheet_name', os.path.basename(os.path.abspath(sheet_path))))

        sheet_properties = sheet_conf.get('sheet_properties', [])

        sheet.write(0, 0, 'Properties:')

        sheet.write(0, 1,
          'N/A' if len(sheet_properties) == 0 else
          ' '.join(str(k) + '=' + repr(v) for (k, v) in sheet_properties))

        sheet.row(2).height = 800
        sheet.panes_frozen = True
        sheet.remove_splits = True

        sheet_comparisons = sheet_conf.get('comparisons', [])

        row = 2

        col = 0

        for (w, caption) in [
                (2500, 'Module'),
                (10000, 'Test'),
                (2000, 'Image\nwidth'),
                (2000, 'Image\nheight'),
                (2000, 'Data\ntype'),
                (7500, 'Other parameters')]:
            sheet.col(col).width = w
            if args.show_times_per_pixel:
                sheet.write_merge(row, row + 1, col, col, caption, header_style)
            else:
                sheet.write(row, col, caption, header_style)
            col += 1

        for config_name in config_names:
            if args.show_times_per_pixel:
                sheet.col(col).width = 3000
                sheet.col(col + 1).width = 3000
                sheet.write_merge(row, row, col, col + 1, config_name, header_style)
                sheet.write(row + 1, col, 'total, ms', subheader_style)
                sheet.write(row + 1, col + 1, 'per pixel, ns', subheader_style)
                col += 2
            else:
                sheet.col(col).width = 4000
                sheet.write(row, col, config_name, header_style)
                col += 1

        col += 1 # blank column between configurations and comparisons

        for comp in sheet_comparisons:
            sheet.col(col).width = 4000
            caption = comp['to'] + '\nvs\n' + comp['from']
            if args.show_times_per_pixel:
                sheet.write_merge(row, row + 1, col, col, caption, header_style)
            else:
                sheet.write(row, col, caption, header_style)
            col += 1

        row += 2 if args.show_times_per_pixel else 1

        sheet.horz_split_pos = row
        sheet.horz_split_first_visible = row

        module_colors = sheet_conf.get('module_colors', {})
        module_styles = {module: xlwt.easyxf('pattern: pattern solid, fore_color {}'.format(color))
                         for module, color in module_colors.iteritems()}

        for module, tests in sorted(collector.tests.iteritems()):
            for ((test, param), configs) in sorted(tests.iteritems()):
                sheet.write(row, 0, module, module_styles.get(module, xlwt.Style.default_style))
                sheet.write(row, 1, test)

                param_list = param[1:-1].split(', ') if param.startswith('(') and param.endswith(')') else [param]

                image_size = next(ifilter(re_image_size.match, param_list), None)
                if image_size is not None:
                    (image_width, image_height) = map(int, image_size.split('x', 1))
                    sheet.write(row, 2, image_width)
                    sheet.write(row, 3, image_height)
                    del param_list[param_list.index(image_size)]

                data_type = next(ifilter(re_data_type.match, param_list), None)
                if data_type is not None:
                    sheet.write(row, 4, data_type)
                    del param_list[param_list.index(data_type)]

                sheet.row(row).write(5, ' | '.join(param_list))

                col = 6

                for c in config_names:
                    if c in configs:
                        sheet.write(row, col, configs[c], time_style)
                    else:
                        sheet.write(row, col, None, no_time_style)
                    col += 1
                    if args.show_times_per_pixel:
                        sheet.write(row, col,
                          xlwt.Formula('{0} * 1000000 / ({1} * {2})'.format(
                              xlwt.Utils.rowcol_to_cell(row, col - 1),
                              xlwt.Utils.rowcol_to_cell(row, 2),
                              xlwt.Utils.rowcol_to_cell(row, 3)
                          )),
                          time_style
                        )
                        col += 1

                col += 1 # blank column

                for comp in sheet_comparisons:
                    cmp_from = configs.get(comp["from"])
                    cmp_to = configs.get(comp["to"])

                    if isinstance(cmp_from, numbers.Number) and isinstance(cmp_to, numbers.Number):
                        try:
                            speedup = cmp_from / cmp_to
                            sheet.write(row, col, speedup, good_speedup_style if speedup > 1.1 else
                                                           bad_speedup_style  if speedup < 0.9 else
                                                           speedup_style)
                        except ArithmeticError as e:
                            sheet.write(row, col, None, error_speedup_style)
                    else:
                        sheet.write(row, col, None, no_speedup_style)

                    col += 1

                row += 1
                if row % 1000 == 0: sheet.flush_row_data()

    wb.save(args.output)
Ejemplo n.º 60
0
    G = WCFG()
    processed = set()

    def make_rules(lhs, start, end):
        if (start, lhs, end) in processed:
            return
        processed.add((lhs, start, end))
        for item in agenda.itercomplete(lhs, start, end):
            G.add(get_intersected_rule(item))
            fsa_states = item.inner + (item.dot, )
            for i, sym in itertools.ifilter(lambda (_, s): is_nonterminal(s),
                                            enumerate(item.rule.rhs)):
                if (
                        sym, fsa_states[i], fsa_states[i + 1]
                ) not in processed:  # Nederhof does not perform this test, but in python it turned out crucial
                    make_rules(sym, fsa_states[i], fsa_states[i + 1])

    # create goal items
    for start, ends in agenda.itergenerating(root):
        if not fsa.is_initial(start):
            continue
        for end in itertools.ifilter(lambda q: fsa.is_final(q), ends):
            make_rules(root, start, end)
            final_weight = fsa.get_final_weight(end)
            G.add(
                Rule(make_symbol(goal, None, None),
                     [make_symbol(root, start, end)], final_weight))

    return G