def test_update(self):
        self.assertRaises(TypeError,
                          OrderedDict().update, [('a', 1), ('b', 2)],
                          None)  # too many args
        pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)]
        od = OrderedDict()
        od.update(dict(pairs))
        self.assertEqual(sorted(od.items()), pairs)  # dict input
        od = OrderedDict()
        od.update(**dict(pairs))
        self.assertEqual(sorted(od.items()), pairs)  # kwds input
        od = OrderedDict()
        od.update(pairs)
        self.assertEqual(list(od.items()), pairs)  # pairs input
        od = OrderedDict()
        od.update([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5)
        self.assertEqual(list(od.items()), pairs)  # mixed input

        # Make sure that direct calls to update do not clear previous contents
        # add that updates items are not moved to the end
        d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
        d.update([('e', 5), ('f', 6)], g=7, d=4)
        self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3),
                                           ('d', 4), ('e', 5), ('f', 6),
                                           ('g', 7)])
def split_sets(split, response_sets, survey_cache, split_entities = False):
    result = []

    if split:
        for ds in split:
            result.append((ds, flatten_response_queryset(response_sets.filter(data_series=ds), survey_cache)))
    else:
        split = [None]
        result.append((None, flatten_response_queryset(response_sets, survey_cache)))

    if split_entities:
        result_dict = OrderedDict()
        series = []
        for ds, qs in result:
            series.append(ds)

        for ds, qs in result:
            filter_dict = OrderedDict()
            for rs in qs:
                #Assign all responsesets to the entity that owns them
                filter_dict.setdefault(rs.entity, []).append(rs)

            for entity, data in filter_dict.items():
                if entity not in result_dict.keys():
                    for s in series:
                        # Ensure all dataseries are present in entity output
                        result_dict.setdefault(entity, OrderedDict())[s] = []
                result_dict[entity][ds] = data
        for entity, data in result_dict.items():
            result_dict[entity] = data.items()
        result = result_dict
    return result
    def run(self):
        results = OrderedDict()
        for key, qs in self.get_responsesets().items():
            if isinstance(qs, list):
                if len(qs) == 0:
                    results[key] = plugins.Vector([])
                    continue

                results[key] = self.scorecard.get_values(qs)

            if isinstance(qs,dict):
                result = []
                for entity, data in qs.items():
                    result.append((entity, self.scorecard.get_values(data)))
                results[key] = plugins.Vector(result)


        if self.reverse_axis:
            if self.aggregate_on and self.aggregate_by_entity:
                for entity, data in results.items():
                    flipped = OrderedDict()
                    for key, scorecard in data.get_values():
                        for operation, values in scorecard:
                            flipped[operation] = flipped.get(operation, [])
                            flipped[operation].append((key, values))
                    results[entity] = plugins.Vector(flipped.items())
            else:
                flipped = OrderedDict()
                for key, scorecard in results.items():
                    for operation, values in scorecard:
                        flipped[operation] = flipped.get(operation, [])
                        flipped[operation].append((key, values))
                results = flipped

        return results
 def test_setdefault(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     pair_order = list(od.items())
     self.assertEqual(od.setdefault('a', 10), 3)
     # make sure order didn't change
     self.assertEqual(list(od.items()), pair_order)
     self.assertEqual(od.setdefault('x', 10), 10)
     # make sure 'x' is added to the end
     self.assertEqual(list(od.items())[-1], ('x', 10))
 def test_setdefault(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     pair_order = list(od.items())
     self.assertEqual(od.setdefault('a', 10), 3)
     # make sure order didn't change
     self.assertEqual(list(od.items()), pair_order)
     self.assertEqual(od.setdefault('x', 10), 10)
     # make sure 'x' is added to the end
     self.assertEqual(list(od.items())[-1], ('x', 10))
 def test_delitem(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     od = OrderedDict(pairs)
     del od['a']
     self.assert_('a' not in od)
     self.assertRaises(KeyError, od.__delitem__, 'a')
     self.assertEqual(list(od.items()), pairs[:2] + pairs[3:])
    def update_metadata(self, new_metadata):
        if not new_metadata:
            return
        has_changes = False
        for key, val in new_metadata.items():
            if self.metadata.get(key) != val:
                has_changes = True
        if not has_changes:
            return
        data = deepcopy(self.metadata)
        data.update(new_metadata)
        data = OrderedDict(sorted(data.items(), key=lambda t: t[0]))        

        new_json = json.dumps(data, indent=4)
        

        org_content = _read_unicode_file_dammit(self.full_local_path)

        def replacer(m):
            parts = m.group(0).split('\n')
            return parts[0].strip() + '\n' + new_json + '\n' + parts[-1].strip()

        new_content = self._json_comment_re.sub(replacer, org_content)
        if new_content == org_content:
            return
        logger.debug("Writing new metadata")
        f = codecs.open(self.full_local_path, 'w', 'utf-8')
        f.write(new_content)
        f.close()
    def test_init(self):
        self.assertRaises(TypeError, OrderedDict, ([('a', 1), ('b', 2)], None))
        # too many args
        pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)]
        self.assertEqual(sorted(OrderedDict(dict(pairs)).items()),
                         pairs)  # dict input
        self.assertEqual(sorted(OrderedDict(**dict(pairs)).items()),
                         pairs)  # kwds input
        self.assertEqual(list(OrderedDict(pairs).items()),
                         pairs)  # pairs input
        self.assertEqual(
            list(
                OrderedDict([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3,
                            e=5).items()), pairs)  # mixed input

        # make sure no positional args conflict with possible kwdargs
        self.assertEqual(
            inspect.getargspec(OrderedDict.__dict__['__init__'])[0], ['self'])

        # Make sure that direct calls to __init__ do not clear previous contents
        d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
        d.__init__([('e', 5), ('f', 6)], g=7, d=4)
        self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3),
                                           ('d', 4), ('e', 5), ('f', 6),
                                           ('g', 7)])
Exemple #9
0
    def get(self, row, columns=None, column_start=None, super_column=None, column_finish=None, column_count=100):
        try:
            if super_column is None:
                data_columns = self.data[row]
            else:
                data_columns = self.data[row][super_column]
            results = OrderedDict()
            count = 0
            if columns is not None:
                for c in columns:
                    results[c] = data_columns[c]
            else:
                for c in sorted(data_columns.keys()):
                    if count > column_count:
                        break
                    if column_start and cmp(c,column_start) < 0:
                        continue
                    if column_finish and cmp(c, column_finish) > 0:
                        break

                    results[c] = data_columns[c]
                    count += 1
            if not len(results):
                raise NotFoundException
            for key, value in results.items():
                if isinstance(value, dict) and len(value) == 0:
                    del(results[key])
                if value is None:
                    del(results[key])
            return results
        except KeyError:
            raise NotFoundException
Exemple #10
0
    def update_metadata(self, new_metadata):
        if not new_metadata:
            return
        print "Update metadata?"
        has_changes = False
        for key, val in new_metadata.items():
            if self.metadata.get(key) != val:
                has_changes = True
        if not has_changes:
            return
        print "Has changes"
        data = deepcopy(self.metadata)
        data.update(new_metadata)
        data = OrderedDict(sorted(data.items(), key=lambda t: t[0]))        

        new_json = json.dumps(data, indent=4)
        
        f = codecs.open(self.full_local_path, 'r', 'utf-8')
        org_content = f.read()
        f.close()
        def replacer(m):
            parts = m.group(0).split('\n')
            return parts[0].strip() + '\n' + new_json + '\n' + parts[-1].strip()

        new_content = self._json_comment_re.sub(replacer, org_content)
        print new_content
        if new_content == org_content:
            return
        print "Write new content"
        f = codecs.open(self.full_local_path, 'w', 'utf-8')
        f.write(new_content)
        f.close()
class ProcessTerminationMonitor(plugins.Observable):
    def __init__(self):
        plugins.Observable.__init__(self)
        self.processesForKill = OrderedDict()
        self.exitHandlers = OrderedDict()

    def listRunningProcesses(self):
        processesToCheck = guiConfig.getCompositeValue("query_kill_processes", "", modeDependent=True)
        if "all" in processesToCheck:
            processesToCheck = [".*"]
        if len(processesToCheck) == 0:
            return []

        running = []
        triggerGroup = plugins.TextTriggerGroup(processesToCheck)
        for process, description in self.processesForKill.values():
            if triggerGroup.stringContainsText(description):
                running.append("PID " + str(process.pid) + " : " + description)

        return running

    def getProcessIdentifier(self, process):
        # Unfortunately the child_watch_add method needs different ways to
        # identify the process on different platforms...
        if os.name == "posix":
            return process.pid
        else:
            return process._handle

    def startProcess(
        self, cmdArgs, description="", killOnTermination=True, exitHandler=None, exitHandlerArgs=(), **kwargs
    ):
        process = subprocess.Popen(cmdArgs, stdin=open(os.devnull), **kwargs)
        pidOrHandle = self.getProcessIdentifier(process)
        self.exitHandlers[int(pidOrHandle)] = (exitHandler, exitHandlerArgs)
        if killOnTermination:
            self.processesForKill[int(pidOrHandle)] = (process, description)
        gobject.child_watch_add(pidOrHandle, self.processExited)

    def processExited(self, pid, *args):
        if self.processesForKill.has_key(pid):
            del self.processesForKill[pid]

        if self.exitHandlers.has_key(pid):
            exitHandler, exitHandlerArgs = self.exitHandlers.pop(pid)
            if exitHandler:
                exitHandler(*exitHandlerArgs)

    def notifyKillProcesses(self, sig=None):
        # Don't leak processes
        if len(self.processesForKill) == 0:
            return
        diag = logging.getLogger("kill processes")
        self.notify("Status", "Terminating all external viewers ...")
        for pid, (process, description) in self.processesForKill.items():
            if self.exitHandlers.has_key(pid):
                self.exitHandlers.pop(pid)  # don't call exit handlers in this case, we're terminating
            self.notify("ActionProgress")
            diag.info("Killing '" + description + "' interactive process")
            killSubProcessAndChildren(process, sig)
Exemple #12
0
    def update_metadata(self, new_metadata):
        if not new_metadata:
            return
        has_changes = False
        for key, val in new_metadata.items():
            if self.metadata.get(key) != val:
                has_changes = True
        if not has_changes:
            return
        data = deepcopy(self.metadata)
        data.update(new_metadata)
        data = OrderedDict(sorted(data.items(), key=lambda t: t[0]))

        new_json = json.dumps(data, indent=4)

        org_content = _read_unicode_file_dammit(self.full_local_path)

        def replacer(m):
            parts = m.group(0).split('\n')
            return parts[0].strip() + '\n' + new_json + '\n' + parts[-1].strip(
            )

        new_content = self._json_comment_re.sub(replacer, org_content)
        if new_content == org_content:
            return
        logger.debug("Writing new metadata")
        f = codecs.open(self.full_local_path, 'w', 'utf-8')
        f.write(new_content)
        f.close()
 def test_delitem(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     od = OrderedDict(pairs)
     del od['a']
     self.assert_('a' not in od)
     self.assertRaises(KeyError, od.__delitem__, 'a')
     self.assertEqual(list(od.items()), pairs[:2] + pairs[3:])
Exemple #14
0
 def get_field_names_for_question_set(questions, respondent_set):
     fields = OrderedDict()
     for qu in questions:
         field_names = CsvFieldGenerator.get_field_names_for_question(qu, respondent_set)
         if field_names is not None:
             fields = OrderedDict(fields.items() + field_names.items())
     return fields
Exemple #15
0
 def __new__(cls, name, bases, attrs):
     attrs['prefix'] = attrs.get('prefix', 'form')
     attrs = OrderedDict(
         sorted([(k, v) for k, v in attrs.items()],
                cmp=lambda x, y: cmp(getattr(x[1], 'order_counter', None),
                                     getattr(y[1], 'order_counter', None))))
     return validators.DeclarativeMeta.__new__(cls, name, bases, attrs)
Exemple #16
0
def test_form(req):
    # get_var_info = {'len': len(args)}
    var_info = OrderedDict((
        ('form_keys', req.form.keys()),
        ('form_values', ', '.join(tuple(uni(req.form.getvalue(key))
                                  for key in req.form.keys()))),
        ('form_getfirst', '%s,%s' % (req.form.getfirst('pname'),
                                     req.form.getfirst('px'))),
        ('form_getlist', '%s,%s' % (req.form.getlist('pname'),
                                    req.form.getlist('px'))),
        ('', ''),
        ('args_keys', req.args.keys()),
        ('args_values', ', '.join(tuple(uni(req.args[key])
                                        for key in req.args.keys()))),
        ('args_getfirst', '%s,%s' % (req.args.getfirst('gname'),
                                     req.args.getfirst('gx'))),
        ('args_getlist', '%s,%s' % (req.args.getlist('gname'),
                                    req.args.getlist('gx'))),
        ))

    buff = get_header("HTTP Form args test") + \
        (get_crumbnav(req),
         "<h2>Get Form</h2>",
         '<form method="get">',
         '<input type="text" name="gname" value="Ondřej"><br/>',
         '<input type="text" name="gsurname" value="Tůma"><br/>',
         '<input type="text" name="gx" value="1">'
         '<input type="text" name="gx" value="2">'
         '<input type="text" name="gx" value="3"><br/>',
         '<input type="submit" name="btn" value="Send">'
         '</form>',
         "<h2>Post Form</h2>",
         '<form method="post">',
         '<input type="text" name="pname" value="Ondřej"><br/>',
         '<input type="text" name="psurname" value="Tůma"><br/>',
         '<input type="text" name="px" value="8">'
         '<input type="text" name="px" value="7">'
         '<input type="text" name="px" value="6"><br/>',
         '<input type="submit" name="btn" value="Send">'
         '</form>',
         "<h2>Variables</h2>",
         "<table>") + \
        tuple("<tr><td>%s:</td><td>%s</td></tr>" %
              (key, html(val)) for key, val in var_info.items()) + \
        ("</table>",
         "<h2>Browser Headers</h2>",
         "<table>") + \
        tuple("<tr><td>%s:</td><td>%s</td></tr>" %
              (key, val) for key, val in req.headers_in.items()) + \
        ("</table>",
         "<h2>Request Variables </h2>",
         "<table>") + \
        tuple("<tr><td>%s:</td><td>%s</td></tr>" %
              (key, val) for key, val in get_variables(req)) + \
        ("</table>",) + \
        get_footer()

    for line in buff:
        req.write(line + '\n')
    return state.OK
Exemple #17
0
def get_book_number(phrase):
	""" Find book number of requested book of the Bible. Used to handle multiple forms
	of writing a certain Bible book. """
	
	book_names = OrderedDict([('genesis',1) , ('gen',1) , ('gn',1), ('bereshit', 1), ('exodus',2), ('exod',2), ('ex',2), ('shemot',2), ('leviticus',3),
						 ('lev',3), ('lv',3), ('vayikra',3), ('numbers',4), ('num',4), ('nm',4), ('bemidbar',4), ('deuteronomy',5), ('deut',5),
						 ('dt',5), ('devarim',5), ('joshua',6), ('josh',6), ('yehoshua',6), ('judges',7), ('judg',7), ('jgs',7), ('shoftim',7), ('ruth',8), ('ru',8),
						 ('1 samuel',9), ('1samuel',9), ('1 sam',9), ('1sam',9), ('1 sm',9), ('1sm',9), ('1 shmuel',9), ('1shmuel',9), ('2 samuel',10),
						 ('2samuel',10), ('2 sam',10), ('2sam',10), ('2 sm',10), ('2sm',10), ('2 shmuel',10), ('2shmuel',10), ('1 kings',11), ('1kings',11),
						 ('1 kgs',11), ('1kgs',11), ('1 melachim',11), ('1melachim',11), ('2 kings',12), ('2kings',12), ('2 kgs',12), ('2kgs',12), ('2 melachim',12), ('2melachim',12),
						 ('1 chronicles',13), ('1chronicles',13), ('1 chron',13), ('1chron',13), ('1 chr',13), ('1chr',13), ('2 chronicles',14),
						 ('2chronicles',14), ('2 chron',14), ('2chron',14), ('2 chr',14), ('2chr',14), ('ezra',15), ('ezr',15), ('nehemiah',16),
						 ('neh',16), ('esther',17), ('est',17), ('job',18), ('jb',18), ('psalms',19), ('psalm',19), ('ps',19),
						 ('pss',19), ('proverbs',20), ('prov',20), ('prv',20), ('ecclesiastes',21), ('eccles',21), ('eccl',21),
						 ('song of solomon',22), ('song of songs',22), ('song of sol',22), ('sg',22), ('isaiah',23), ('isa',23), ('yeshayahu',23), ('yeshaya',23),
						 ('jeremiah',24), ('jer',24), ('yirmiyahu',24), ('yirmiyah',24), ('lamentations',25), ('lam',25), ('ezekiel',26), ('ezek',26), ('yechezkel',26), ('daniel',27),
						 ('dan',27), ('dn',27), ('hosea',28), ('hos',28), ('hoshea',28), ('joel',29), ('jl',29), ('yoel',29), ('amos',30), ('am',30),
						 ('obadiah',31), ('obad',31), ('ob',31), ('ovadiah',31), ('ovadyah',31), ('jonah',32), ('jon',32), ('micah',33), ('mic',33), ('michah',33), ('nahum',34),
						 ('nah',34), ('na',34), ('nachum',34), ('habakkuk',35), ('hab',35), ('hb',35), ('chavakuk',35), ('zephaniah',36), ('zeph',36), ('zep',36), ('tzefaniah',36), ('tzefanyah',36),
						 ('haggai',37), ('hag',37), ('hg',37), ('chaggai',37), ('zechariah',38), ('zech',38), ('zec',38), ('zecharya',38), ('zecharyah',38), ('zechariyah',38), ('malachi',39),
						 ('mal',39), ('matthew',40), ('mathew',40), ('matt',40), ('mat',40), ('mt',40), ('mark',41), ('mk',41), ('luke',42), ('lk',42),
						 ('john',43), ('jn',43), ('acts',44), ('acts of the apostles',44), ('romans',45), ('rom',45),
						 ('1 corinthians',46), ('1corinthians',46), ('1 cor',46), ('1cor',46), ('2 corinthians',47), ('2corinthians',47),
						 ('2 cor',47), ('2cor',47), ('galatians',48), ('gal',48), ('ephesians',49), ('eph',49), ('philippians',50), ('phil',50),
						 ('colossians',51), ('col',51), ('1 thessalonians',52), ('1thessalonians',52), ('1 thess',52), ('1thess',52),
						 ('1 thes',52), ('1thes',52), ('2 thessalonians',53), ('2thessalonians',53), ('2 thess',53), ('2thess',53),
						 ('2 thes',53), ('2thes',53), ('1 timothy',54), ('1timothy',54), ('1st timothy',54), ('1 tim',54), ('1tim',54), ('1 tm',54), ('1tm',54),
						 ('2 timothy', 55), ('2timothy',55), ('2nd timothy',55), ('2 tim',55), ('2tim',55), ('2 tm',55), ('2tm',55), ('titus',56), ('ti',56),
						 ('philemon',57), ('philem',57), ('phlm',57), ('hebrews',58), ('heb',58), ('james',59), ('jas',59),
						 ('1 peter',60), ('1peter',60), ('1 pet',60), ('1pet',60), ('1 pt',60), ('1pt',60), ('2 peter',61), ('2peter',61),
						 ('2 pet',61), ('2pet',61), ('2 pt',61), ('2pt',61), ('1 john',62), ('1john',62), ('1 jn',62), ('1jn',62), ('2 john',63),
						 ('2john',63), ('2 jn',63), ('2jn',63), ('3 john',64), ('3john',64), ('3 jn',64), ('3jn',64), ('jude',65), ('revelation',66),
						 ('revelations',66), ('rev',66), ('rv',66), ('judith',67), ('judeth',67), ('jdt',67), ('wisdom',68), ('wis',68), ('wisdom of solomon', 68),
						 ('tobit',69), ('tob',69), ('sirach',70), ('sir',70), ('ecclesiasticus',70), ('baruch',71), ('bar',71), ('1 maccabees',72),
						 ('1maccabees',72), ('1 macc',72), ('1macc',72), ('1 mac',72), ('1mac', 72), ('2 maccabees',73), ('2maccabees',73), ('2 macc', 73),
						 ('2macc',73), ('2 mac',73), ('2mac',73), ('3 maccabees',74), ('3maccabees',74), ('3 macc',74), ('3macc',74), ('3 mac',74), ('3mac',74),
						 ('4 maccabees',75), ('4maccabees',75), ('4 macc',75), ('4macc',75), ('4 mac',75), ('4mac',75), ('rest of daniel',76), ('additions to daniel',76),
						 ('adddan',76), ('song of the three children',76), ('prayer of azariah',76), ('rest of esther',77), ('additions to esther',77), ('addesth',77),
						 ('prayer of manasses',78), ('prayer of manasseh',78), ('manasses',78), ('manasseh',78), ('prman',78), ('1 esdras',79), ('1esdras',79), ('1 esd',79),
						 ('1esd',79), ('2 esdras',80), ('2esdras',80), ('2 esd',80), ('2esd',80), ('story of susanna',81), ('susanna',81), ('sus',81), ('bel and the dragon',82), ('bel',82)])

	sorted_books = OrderedDict(sorted(book_names.items(), key=lambda t: len(t[0]), reverse=True))
	for key, value in sorted_books.items():
		if search(r'\b' + key + r'\b', phrase):
			return value
	return False
Exemple #18
0
def get_book_number(phrase):
	""" Find book number of requested book of the Bible. Used to handle multiple forms
	of writing a certain Bible book. """
	
	book_names = OrderedDict([('genesis',1) , ('gen',1) , ('gn',1), ('bereshit', 1), ('exodus',2), ('exod',2), ('ex',2), ('shemot',2), ('leviticus',3),
                         ('lev',3), ('lv',3), ('vayikra',3), ('numbers',4), ('num',4), ('nm',4), ('bemidbar',4), ('deuteronomy',5), ('deut',5),
                         ('dt',5), ('devarim',5), ('joshua',6), ('josh',6), ('yehoshua',6), ('judges',7), ('judg',7), ('jgs',7), ('shoftim',7), ('ruth',8), ('ru',8),
                         ('1 samuel',9), ('1samuel',9), ('1 sam',9), ('1sam',9), ('1 sm',9), ('1sm',9), ('1 shmuel',9), ('1shmuel',9), ('2 samuel',10),
                         ('2samuel',10), ('2 sam',10), ('2sam',10), ('2 sm',10), ('2sm',10), ('2 shmuel',10), ('2shmuel',10), ('1 kings',11), ('1kings',11),
                         ('1 kgs',11), ('1kgs',11), ('1 melachim',11), ('1melachim',11), ('2 kings',12), ('2kings',12), ('2 kgs',12), ('2kgs',12), ('2 melachim',12), ('2melachim',12),
                         ('1 chronicles',13), ('1chronicles',13), ('1 chron',13), ('1chron',13), ('1 chr',13), ('1chr',13), ('2 chronicles',14),
                         ('2chronicles',14), ('2 chron',14), ('2chron',14), ('2 chr',14), ('2chr',14), ('ezra',15), ('ezr',15), ('nehemiah',16),
                         ('neh',16), ('esther',17), ('est',17), ('job',18), ('jb',18), ('psalms',19), ('psalm',19), ('ps',19),
                         ('pss',19), ('proverbs',20), ('prov',20), ('prv',20), ('ecclesiastes',21), ('eccles',21), ('eccl',21),
                         ('song of solomon',22), ('song of songs',22), ('song of sol',22), ('sg',22), ('isaiah',23), ('isa',23), ('yeshayahu',23), ('yeshaya',23),
                         ('jeremiah',24), ('jer',24), ('yirmiyahu',24), ('yirmiyah',24), ('lamentations',25), ('lam',25), ('ezekiel',26), ('ezek',26), ('yechezkel',26), ('daniel',27),
                         ('dan',27), ('dn',27), ('hosea',28), ('hos',28), ('hoshea',28), ('joel',29), ('jl',29), ('yoel',29), ('amos',30), ('am',30),
                         ('obadiah',31), ('obad',31), ('ob',31), ('ovadiah',31), ('ovadyah',31), ('jonah',32), ('jon',32), ('micah',33), ('mic',33), ('michah',33), ('nahum',34),
                         ('nah',34), ('na',34), ('nachum',34), ('habakkuk',35), ('hab',35), ('hb',35), ('chavakuk',35), ('zephaniah',36), ('zeph',36), ('zep',36), ('tzefaniah',36), ('tzefanyah',36),
                         ('haggai',37), ('hag',37), ('hg',37), ('chaggai',37), ('zechariah',38), ('zech',38), ('zec',38), ('zecharya',38), ('zecharyah',38), ('zechariyah',38), ('malachi',39),
                         ('mal',39), ('matthew',40), ('mathew',40), ('matt',40), ('mat',40), ('mt',40), ('mark',41), ('mk',41), ('luke',42), ('lk',42),
                         ('john',43), ('jn',43), ('acts',44), ('acts of the apostles',44), ('romans',45), ('rom',45),
                         ('1 corinthians',46), ('1corinthians',46), ('1 cor',46), ('1cor',46), ('2 corinthians',47), ('2corinthians',47),
                         ('2 cor',47), ('2cor',47), ('galatians',48), ('gal',48), ('ephesians',49), ('philippians',50), ('phil',50),
                         ('colossians',51), ('col',51), ('1 thessalonians',52), ('1thessalonians',52), ('1 thess',52), ('1thess',52),
                         ('1 thes',52), ('1thes',52), ('2 thessalonians',53), ('2thessalonians',53), ('2 thess',53), ('2thess',53),
                         ('2 thes',53), ('2thes',53), ('1 timothy',54), ('1timothy',54), ('1st timothy',54), ('1 tim',54), ('1tim',54), ('1 tm',54), ('1tm',54),
                         ('2 timothy', 55), ('2timothy',55), ('2nd timothy',55), ('2 tim',55), ('2tim',55), ('2 tm',55), ('2tm',55), ('titus',56), ('ti',56),
                         ('philemon',57), ('philem',57), ('phlm',57), ('hebrews',58), ('heb',58), ('james',59), ('jas',59),
                         ('1 peter',60), ('1peter',60), ('1 pet',60), ('1pet',60), ('1 pt',60), ('1pt',60), ('2 peter',61), ('2peter',61),
                         ('2 pet',61), ('2pet',61), ('2 pt',61), ('2pt',61), ('1 john',62), ('1john',62), ('1 jn',62), ('1jn',62), ('2 john',63),
                         ('2john',63), ('2 jn',63), ('2jn',63), ('3 john',64), ('3john',64), ('3 jn',64), ('3jn',64), ('jude',65), ('revelation',66),
                         ('revelations',66), ('rev',66), ('rv',66), ('judith',67), ('judeth',67), ('jdt',67), ('wisdom',68), ('wis',68), ('wisdom of solomon', 68),
                         ('tobit',69), ('tob',69), ('sirach',70), ('sir',70), ('ecclesiasticus',70), ('baruch',71), ('bar',71), ('1 maccabees',72),
                         ('1maccabees',72), ('1 macc',72), ('1macc',72), ('1 mac',72), ('1mac', 72), ('2 maccabees',73), ('2maccabees',73), ('2 macc', 73),
                         ('2macc',73), ('2 mac',73), ('2mac',73), ('3 maccabees',74), ('3maccabees',74), ('3 macc',74), ('3macc',74), ('3 mac',74), ('3mac',74),
                         ('4 maccabees',75), ('4maccabees',75), ('4 macc',75), ('4macc',75), ('4 mac',75), ('4mac',75), ('rest of daniel',76), ('additions to daniel',76),
                         ('adddan',76), ('song of the three children',76), ('prayer of azariah',76), ('rest of esther',77), ('additions to esther',77), ('addesth',77),
                         ('prayer of manasses',78), ('prayer of manasseh',78), ('manasses',78), ('manasseh',78), ('prman',78), ('1 esdras',79), ('1esdras',79), ('1 esd',79),
                         ('1esd',79), ('2 esdras',80), ('2esdras',80), ('2 esd',80), ('2esd',80), ('story of susanna',81), ('susanna',81), ('sus',81), ('bel and the dragon',82), ('bel',82)])

	sorted_books = OrderedDict(sorted(book_names.items(), key=lambda t: len(t[0]), reverse=True))
	for key, value in sorted_books.items():
		if search(r'\b' + key + r'\b', phrase):
			return value
	return False
 def test_iterators(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     self.assertEqual(list(od), [t[0] for t in pairs])
     self.assertEqual(list(od.keys()), [t[0] for t in pairs])
     self.assertEqual(list(od.values()), [t[1] for t in pairs])
     self.assertEqual(list(od.items()), pairs)
     self.assertEqual(list(reversed(od)), [t[0] for t in reversed(pairs)])
 def test_reinsert(self):
     # Given insert a, insert b, delete a, re-insert a,
     # verify that a is now later than b.
     od = OrderedDict()
     od['a'] = 1
     od['b'] = 2
     del od['a']
     od['a'] = 1
     self.assertEqual(list(od.items()), [('b', 2), ('a', 1)])
 def test_reinsert(self):
     # Given insert a, insert b, delete a, re-insert a,
     # verify that a is now later than b.
     od = OrderedDict()
     od['a'] = 1
     od['b'] = 2
     del od['a']
     od['a'] = 1
     self.assertEqual(list(od.items()), [('b', 2), ('a', 1)])
Exemple #22
0
 def getFreeTextData(self, tests):
     data = OrderedDict()
     for testName, state, extraVersion in tests:
         freeText = state.freeText if hasattr(state, "freeText") else None
         if freeText:
             if not data.has_key(freeText):
                 data[freeText] = []
             data[freeText].append((testName, state, extraVersion))
     return data.items()
 def getFreeTextData(self):
     data = OrderedDict()
     for test in self.getAllTests():
         freeText = test.state.freeText
         if freeText:
             if not data.has_key(freeText):
                 data[freeText] = []
             data[freeText].append(test)
     return data.items()
 def _get_tab(self, first_id, last_id):
     """Return a list of selected items in a tab section of settings."""
     selected_items = {}
     for id in range(first_id, last_id + 1):
         if self._xbmcaddon.getSetting(str(id)) == "true":
             selected_items.update({id: KEYWORDS[str(id)]})
     # Sort dictionary by key.
     selected_items = OrderedDict(sorted(selected_items.items()))
     return selected_items
 def getFreeTextData(self, tests):
     data = OrderedDict()
     for testName, state, extraVersion in tests:
         freeText = state.freeText
         if freeText:
             if not data.has_key(freeText):
                 data[freeText] = []
             data[freeText].append((testName, state, extraVersion))
     return data.items()
 def test_iterators(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     self.assertEqual(list(od), [t[0] for t in pairs])
     self.assertEqual(list(od.keys()), [t[0] for t in pairs])
     self.assertEqual(list(od.values()), [t[1] for t in pairs])
     self.assertEqual(list(od.items()), pairs)
     self.assertEqual(list(reversed(od)),
                      [t[0] for t in reversed(pairs)])
Exemple #27
0
    def __add_parser(self, config_path, parent=None):
        # Get configuration elements from config_path.
        config = self.__get_config(config_path)

        self.__check_parser_conf(config_path, config)

        # Init main parser.
        if parent is None:
            self.parser = argparse.ArgumentParser(add_help=True)
            parent = self.parser
        # We may need to access to the parser object later (for printing usage
        # by example) so save it to __parsers attibute.
        self.__parsers.setdefault('/'.join(config_path), parent)

        # Add subparser.
        if config.get('subparsers'):
            subparsers_config = config.get('subparsers', {})
            subparser_dest = '%s%d' % (self.keyword, len(config_path) / 2)
            subparsers = parent.add_subparsers(dest=subparser_dest)
            subparsers.required = True
            for subparser_name in subparsers_config:
                subparser_path = list(config_path)
                subparser_path.extend(['subparsers', subparser_name])
                subparser = subparsers.add_parser(subparser_name, add_help=True)
                subparser.name = subparser_name
                self.__add_parser(subparser_path, subparser)
        else:
            # For options, use a deep copy for not altering initial
            # configuration when change must be done.
            options_config = copy.deepcopy(config.get('options', {}))
            args_config = OrderedDict(config.get('args', {}))

            if 'usage' in config:
                parent.usage = self.__usage(parent.prog, config['usage'])
            if 'desc' in config:
                parent.description = config['desc']

            try:
                # Add groups.
                for group_type in 'groups', 'exclusive_groups':
                    if config.get(group_type, {}):
                        self.__add_groups(
                            parent, config.get(group_type), options_config,
                            True if group_type == 'exclusive_groups' else False
                        )

                # Add options.
                for option, option_config in options_config.items():
                    self.__add_option(parent, option, option_config)

                # Add args:
                for arg, arg_config in args_config.items():
                    self.__add_option(parent, arg, arg_config, True)
            except CLGError as err:
                raise CLGError('/%s: %s' % ('/'.join(config_path), err))
Exemple #28
0
def dict2rec(*args, **kwargs):
    """
    Convert dict of arrays to nested recarray.
    
    The input can be anything accepted by :meth:`OrderedDict`, 
    see :class:`dict`. All items must be the same length, which becomes the 
    length of the resulting recarray.
    
    >>> dict2rec(zip(("a", "b", "c"), 
    ...     (np.arange(0, 3), np.arange(10, 13), np.zeros(3))))
    array([(0, 10, 0.0), (1, 11, 0.0), (2, 12, 0.0)], 
          dtype=[('a', '<i...'), ('b', '<i...'), ('c', '<f8')])
    
    >>> dict2rec(OrderedDict(zip(("a", "b", "c"), 
    ...     (np.arange(0, 3), np.arange(10, 13), np.zeros(3)))))
    array([(0, 10, 0.0), (1, 11, 0.0), (2, 12, 0.0)], 
          dtype=[('a', '<i...'), ('b', '<i...'), ('c', '<f8')])
    
    >>> dict2rec({
    ...     "a": np.arange(0, 3), "b": np.arange(10, 13), "c": np.zeros((3,))})
    array([(0, 0.0, 10), (1, 0.0, 11), (2, 0.0, 12)],
          dtype=[('a', '<i...'), ('c', '<f8'), ('b', '<i...')])
    
    Vector-valued fields.
    
    >>> dict2rec([("a", [0, 1, 2]), 
    ...           ("b", [10, 11, 12]), 
    ...           ("c", [(0, 1), (2, 3), (4, 5)]),
    ...          ])
    array([(0, 10, [0, 1]), (1, 11, [2, 3]), (2, 12, [4, 5])], 
          dtype=[('a', '<i4'), ('b', '<i4'), ('c', '<i4', (2,))])
    """
    d = OrderedDict(*args, **kwargs)
    for k, v in d.items():
        d[k] = np.atleast_1d(v)
    # Convert keys to plain str because Numpy field names cannot be unicode
    dtype = [(str(k), v.dtype, v.shape[1:]) for k, v in d.items()]
    shape = len(v)
    x = np.zeros(shape=shape, dtype=dtype)
    for k, v in d.items():
        x[k] = v
    return x
Exemple #29
0
def list_ (request):
	priority = map(int, request.GET.getall('pr'))

	tags = request.GET.get('tags', '')
	def repl (matchobj):
		m = matchobj.group(0)
		if m in ('or', 'and', 'not', '(', ')'):
			return m.upper()
		else:
			return u"tags LIKE '%%#%s#%%'" % m
	tags = re.sub(RE_TAG_Q, repl, tags)

	entries = OrderedDict()
	q = Entry.query.order_by(Entry.priority.desc(), Entry.created_time.desc())
	if tags:
		q = q.filter(tags)
	if priority:
		q = q.filter(Entry.priority.in_(priority))
	for e in q:
		e.children = []
		entries[e.id] = e

	keys_to_del = []
	items = entries.items()
	for id, e in items:
		if e.parent_id:
			keys_to_del.append(id)
			if e.parent_id not in entries: #for filter
				entries[e.parent_id] = e.parent
				e.parent.children = []
				items.append((e.parent_id, e.parent)) #visit parents too later so hierarchy doesnt breake
			entries[e.parent_id].children.append(e)
	for id in keys_to_del:
		del entries[id]

	if tags:
		entries = OrderedDict(sorted(entries.items(), key = (lambda (k, e): e.priority), reverse = True))

	return {
		'entries': entries.values(),
		'priorities': priorities,
	}
Exemple #30
0
    def generate_topic_word(self):
        'generate estimated topics for a document'
        from ordereddict import OrderedDict

        # build topic-vocab map
        topic_vocab = {}
        for word_vocab_id in self.vocab_topic:
            vt = self.vocab_topic[word_vocab_id]
            sorted_vt = OrderedDict(sorted(vt.items(), key=lambda x: -x[1]))
            for svt, (k, v) in enumerate(sorted_vt.items()):
                if svt > 10:
                    break
                else:
                    if k not in topic_vocab:
                        vocab_list = {}
                        vocab_list[word_vocab_id] = v
                    else:
                        vocab_list = topic_vocab[k]
                        if word_vocab_id in vocab_list:
                            old_association = vocab_list[word_vocab_id]
                            if old_association < v:
                                vocab_list[word_vocab_id] = v
                        else:
                            vocab_list[word_vocab_id] = v
                    topic_vocab[k] = vocab_list

        # combine topic-vocab with index_vocab to create a topic_id --> vocabulary list map
        topic_word = {}
        for topic_id in topic_vocab:
            topic_word[topic_id] = []
            tv = topic_vocab[topic_id]
            sorted_tv = OrderedDict(sorted(tv.items(), key=lambda x: -x[1]))
            for stv, (k, v) in enumerate(sorted_tv.items()):
                if stv > 10:
                    break
                else:
                    topic_word[topic_id].append(self.index_vocab[k])
        return topic_word

        '''# publish
Exemple #31
0
 def writeErrors(self, rejectionInfo):
     # Don't write errors if only some of a group are rejected
     appsByName = OrderedDict()
     rejectedApps = set(rejectionInfo.keys())
     for suite in self.suites:
         app = suite.app
         appsByName.setdefault(app.name, []).append(app)
         
     for _, appGroup in appsByName.items():
         if set(appGroup).issubset(rejectedApps):
             for app in appGroup:
                 if app in rejectionInfo:
                     sys.stderr.write(app.rejectionMessage(rejectionInfo.get(app)))
 def findAllClasses(self, className):
     if len(self.allApps) == 0:
         return [ (className, []) ]
     else:
         classNames = OrderedDict()
         for app in self.allApps:
             allConfigsForApp = self.getAllIntvConfigs([ app ])
             replacements = plugins.ResponseAggregator([ x.getReplacements for x in allConfigsForApp])()
             for config in allConfigsForApp:
                 if className in config.getInteractiveActionClasses(self.dynamic):
                     realClassName = replacements.get(className, className)
                     classNames.setdefault(realClassName, []).append(app)
         return classNames.items()
Exemple #33
0
    def writeErrors(self, rejectionInfo):
        # Don't write errors if only some of a group are rejected
        appsByName = OrderedDict()
        rejectedApps = set(rejectionInfo.keys())
        for suite in self.suites:
            app = suite.app
            appsByName.setdefault(app.name, []).append(app)

        for _, appGroup in appsByName.items():
            if set(appGroup).issubset(rejectedApps):
                for app in appGroup:
                    if app in rejectionInfo:
                        sys.stderr.write(app.rejectionMessage(rejectionInfo.get(app)))
Exemple #34
0
def test_upload(req):
    var_info = OrderedDict((
        ('form_keys', req.form.keys()),
        ('form_value_names', ', '.join(tuple(req.form[key].name
                                             for key in req.form.keys()))),
        ('form_value_types', ', '.join(tuple(req.form[key].type
                                             for key in req.form.keys()))),
        ('form_value_fnames', ', '.join(tuple(uni(req.form[key].filename)
                                              for key in req.form.keys()))),
        ('form_value_lenghts', ', '.join(tuple(uni(req.form[key].length)
                                               for key in req.form.keys()))),
        ('form_value_files', ', '.join(tuple(uni(req.form[key].file)
                                             for key in req.form.keys()))),
        ('form_value_lists', ', '.join(tuple(
            'Yes' if req.form[key].list else 'No'
            for key in req.form.keys()))),
        ))

    files = []
    for key in req.form.keys():
        if req.form[key].filename:
            files.append("<h2>%s</h2>" % req.form[key].filename)
            files.append("<i>%s</i>" % req.form[key].type)
            if req.form[key].type.startswith('text/'):
                files.append("<pre>%s</pre>" %
                             html(req.form.getvalue(key).decode('utf-8')))
            else:
                files.append("<pre>%s</pre>" %
                             encodestring(req.form.getvalue(key)).decode())

    buff = get_header('HTTP file upload test') + \
        (get_crumbnav(req),
         "<h2>Upload Form</h2>",
         '<form method="post" enctype="multipart/form-data">',
         '<input type="file" name="file_0"><br/>',
         '<input type="file" name="file_1"><br/>',
         '<input type="file" name="file_2"><br/>',
         '<input type="submit" name="btn" value="Upload">'
         '</form>',
         "<h2>Uploaded File</h2>",
         "<table>") + \
        tuple("<tr><td>%s:</td><td>%s</td></tr>" %
              (key, html(val)) for key, val in var_info.items()) + \
        ("</table>",) + \
        tuple(files) + \
        get_footer()

    for line in buff:
        req.write(line + '\n')
    return state.OK
    def test_update(self):
        self.assertRaises(TypeError, OrderedDict().update, [('a', 1), ('b',
            2)], None)                        # too many args
        pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)]
        od = OrderedDict()
        od.update(dict(pairs))
        self.assertEqual(sorted(od.items()), pairs)                                 # dict input
        od = OrderedDict()
        od.update(**dict(pairs))
        self.assertEqual(sorted(od.items()), pairs)                                 # kwds input
        od = OrderedDict()
        od.update(pairs)
        self.assertEqual(list(od.items()), pairs)                                   # pairs input
        od = OrderedDict()
        od.update([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5)
        self.assertEqual(list(od.items()), pairs)                                   # mixed input

        # Make sure that direct calls to update do not clear previous contents
        # add that updates items are not moved to the end
        d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)])
        d.update([('e', 5), ('f', 6)], g=7, d=4)
        self.assertEqual(list(d.items()),
            [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)])
 def get_folder_items(self):
     folders = OrderedDict()
     items = OrderedDict()
     for item in self.context.listFolderContents():
         item_id = item.getId()
         item_dict = {"id" : item_id,
                      "title": item.Title(),
                      "description": item.Description(),
                      "item_url": item.absolute_url(),
                      "image_url": "" ,
                      "portal_type": item.portal_type}
         if item_dict["portal_type"] in ["Folder",]:
             folders[item_id] = item_dict
             for folder_item in item.listFolderContents():
                 if folder_item.portal_type == "Image":
                     folders[item_id]["image_url"] = \
                         folder_item.absolute_url()
                     break
         elif item_dict["portal_type"] in ["Image"]:
             item_dict["image_url"] = item.absolute_url()
             items[item_id] = item_dict
         else:
             items[item_id] = item_dict
     return OrderedDict(folders.items() + items.items())
Exemple #37
0
 def findAllClasses(self, className):
     if len(self.allApps) == 0:
         return [(className, [])]
     else:
         classNames = OrderedDict()
         for app in self.allApps:
             allConfigsForApp = self.getAllIntvConfigs([app])
             replacements = plugins.ResponseAggregator(
                 [x.getReplacements for x in allConfigsForApp])()
             for config in allConfigsForApp:
                 if className in config.getInteractiveActionClasses(
                         self.dynamic):
                     realClassName = replacements.get(className, className)
                     classNames.setdefault(realClassName, []).append(app)
         return classNames.items()
Exemple #38
0
class DictCache(AbstractCache):
    """Implementation of a cache in a memory dictionary

    """
    def __init__(self, ttl=60):
        """Creates a new instance

        params:

            ``ttl``
                Time to live of the data.

        """
        super(DictCache, self).__init__(ttl=ttl)
        try:
            self._ich = collections.OrderedDict()
            self._ttds = collections.OrderedDict()
        except AttributeError:
            #This version of python does not support OrderedDict
            from ordereddict import OrderedDict
            self._ich = OrderedDict()
            self._ttds = OrderedDict()

    def store_data(self, k, ttl, v):
        self._ich[k] = v
        self._ttds[k] = (
            time.time() + ttl if ttl != None else None
        )

    def retrieve_data(self, k):
        ttd = self._ttds.get(k, 0)
        if ttd == None or time.time() < ttd:
            return self._ich[k]
        elif ttd:
            self._ttds.pop(k)
            self._ich.pop(k)

    def clear_expired(self):
        for k, ttd in self._ttds.items():
            if ttd != None and ttd < time.time():
                self._ttds.pop(k)
                self._ich.pop(k)
            else:
                break

    def clear(self):
        self._ich.clear()
        self._ttds.clear()
 def add_folders(self, folders, sorted_by_values=True):
     """Add many folders to a menu. Can be sorted by folder name."""
     translations = {}
     for id in folders:
         value = self.xbmcaddon.translate(id)
         translations.update({id: value})
     if sorted_by_values:  # Default.
         # IMPORTANT! This is the suggested way by Python 2.7, but it seems
         # to have problems when first letter is accented.
         translations = \
             OrderedDict(sorted(translations.items(), key=lambda t: t[1]))
         for id, folder in translations.iteritems():
             self._add_folder(id, folder, folders[id])
     else:  # Sorted by key.
         for id, folder in folders.iteritems():
             self._add_folder(id, translations[id], folder)
Exemple #40
0
 def begetPathArgs(self, data):
     """ Create pargs for url path generation from pkeys if given
         Do urlencoding
         return pargs dict
     """
     pargs = ODict() 
     # assign values from kwa for pkeys
     for key in self.pkeys:
         try:
             pargs[key] = data[key]
         except KeyError as ex:
             raise KeyEmptorError(msg = "Missing data for path key.",key=key)                
 
     for key, val in pargs.items():
         pargs[key] = urllib.quote(val, '/:;#')
     
     return pargs
Exemple #41
0
def executeCommand(envvars):
    buildenv = os.environ
    env = OrderedDict()
    for name, value in envvars:
        if name == 'COMMAND':
            #commands = value.split(';')
            commands = value;
        else:
            env[name] = value
    handleDollarValues(env)
    for key, value in env.items():
        buildenv[key] = value
    p = subprocess.Popen("%s" % (commands), shell=True, executable = "/bin/sh", env=buildenv)
    sts = os.waitpid(p.pid, 0)[1]
    if ((sts/256) != 0):
        return 0
    return 1
    def expandCollations(self, test):
        newColl = OrderedDict()
        coll = test.getConfigValue("collate_file")
        self.diag.info("coll initial:" + str(coll))
        for targetPattern in sorted(coll.keys()):
            sourcePatterns = coll.get(targetPattern)
            if not glob.has_magic(targetPattern):
                newColl[targetPattern] = sourcePatterns
                continue

            # add each file to newColl by transferring wildcards across
            for sourcePattern in sourcePatterns:
                for sourcePath in self.findPaths(test, sourcePattern):
                    # Use relative paths: easier to debug and SequenceMatcher breaks down if strings are longer than 200 chars
                    relativeSourcePath = plugins.relpath(sourcePath, test.getDirectory(temporary=1))
                    newTargetStem = self.makeTargetStem(targetPattern, sourcePattern, relativeSourcePath)
                    self.diag.info("New collation to " + newTargetStem + " : from " + relativeSourcePath)
                    newColl.setdefault(newTargetStem, []).append(sourcePath)
        return newColl.items()
Exemple #43
0
    def get(self,
            row,
            columns=None,
            column_start=None,
            super_column=None,
            column_finish=None,
            column_count=100):
        try:
            if super_column is None:
                data_columns = self.data[row]
            else:
                data_columns = self.data[row][super_column]
            results = OrderedDict()
            count = 0
            if columns is not None:
                for c in columns:
                    results[c] = data_columns[c]
            else:
                for c in sorted(data_columns.keys()):
                    if count > column_count:
                        break
                    if column_start and cmp(c, column_start) < 0:
                        continue
                    if column_finish and cmp(c, column_finish) > 0:
                        break

                    results[c] = data_columns[c]
                    count += 1
            if not len(results):
                raise NotFoundException
            for key, value in results.items():
                if isinstance(value, dict) and len(value) == 0:
                    del (results[key])
                if value is None:
                    del (results[key])
            return results
        except KeyError:
            raise NotFoundException
 def test_copying(self):
     # Check that ordered dicts are copyable, deepcopyable, picklable,
     # and have a repr/eval round-trip
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     od = OrderedDict(pairs)
     update_test = OrderedDict()
     update_test.update(od)
     for i, dup in enumerate([
             od.copy(),
             copy.copy(od),
             copy.deepcopy(od),
             pickle.loads(pickle.dumps(od, 0)),
             pickle.loads(pickle.dumps(od, 1)),
             pickle.loads(pickle.dumps(od, 2)),
             pickle.loads(pickle.dumps(od, -1)),
             eval(repr(od)),
             update_test,
             OrderedDict(od),
     ]):
         self.assert_(dup is not od)
         self.assertEquals(dup, od)
         self.assertEquals(list(dup.items()), list(od.items()))
         self.assertEquals(len(dup), len(od))
         self.assertEquals(type(dup), type(od))
Exemple #45
0
class CategoryHandler:
    def __init__(self):
        self.testsInCategory = OrderedDict()

    def update(self, categoryHandler):
        for category, testInfo in categoryHandler.testsInCategory.items():
            testInfoList = self.testsInCategory.setdefault(category, [])
            testInfoList += testInfo

    def registerInCategory(self, testId, category, extraVersion, state):
        self.testsInCategory.setdefault(category, []).append((testId, state, extraVersion))

    def getDescription(self, cat, count):
        shortDescr, _ = getCategoryDescription(cat)
        return str(count) + " " + shortDescr

    def getTestCountDescription(self, count):
        return str(count) + " tests: "

    def generateTextSummary(self):
        numTests, summaryData = self.getSummaryData()
        categoryDescs = [ self.getDescription(cat, count) for cat, count in summaryData ]
        return self.getTestCountDescription(numTests) + " ".join(categoryDescs)

    def generateHTMLSummary(self, detailPageRef, extraVersion=None):
        numTests, summaryData = self.getSummaryData(extraVersion)
        container = HTMLgen.Container()
        for cat, count in summaryData:
            summary = HTMLgen.Text(self.getDescription(cat, count))
            if cat == "success":
                container.append(summary)
            else:
                linkTarget = detailPageRef + getCategoryDescription(cat)[-1]
                container.append(HTMLgen.Href(linkTarget, summary))
            
        testCountSummary = HTMLgen.Text(self.getTestCountDescription(numTests))
        return HTMLgen.Container(testCountSummary, container)

    def countTests(self, testInfo, extraVersion):
        if extraVersion is not None:
            return sum((currExtra == extraVersion for (testId, state, currExtra) in testInfo))
        else:
            return len(testInfo)

    def getSummaryData(self, extraVersion=None):
        numTests = 0
        summaryData = []
        for cat, testInfo in self.testsInCategory.items():
            testCount = self.countTests(testInfo, extraVersion)
            if testCount > 0:
                summaryData.append((cat, testCount))
                numTests += testCount
        summaryData.sort(key=self.getSummarySortKey)
        return numTests, summaryData
    
    def getTestsWithDescriptions(self):
        return sorted([ (getCategoryDescription(cat)[1], testInfo) for cat, testInfo in self.testsInCategory.items() ])

    def getSummarySortKey(self, data):
        # Put success at the start, it's neater like that
        return data[0] != "success", -data[1], data[0]
Exemple #46
0
class PluginManager(object):

    ENVIRONMENT_HOOK = 'haas.hooks.environment'

    RESULT_HANDLERS = 'haas.result.handler'

    TEST_RUNNER = 'haas.runner'

    TEST_DISCOVERY = 'haas.discovery'

    _help = {
        TEST_DISCOVERY: 'The test discovery implementation to use.',
        TEST_RUNNER: 'Test runner implementation to use.',
        RESULT_HANDLERS: 'Test result handler implementation to use.',
    }

    _namespace_to_option_parts = {
        TEST_DISCOVERY: ['discovery'],
        TEST_RUNNER: ['runner'],
        RESULT_HANDLERS: ['result', 'handler'],
    }

    def __init__(self):
        self.hook_managers = OrderedDict()
        self.hook_managers[self.ENVIRONMENT_HOOK] = ExtensionManager(
            namespace=self.ENVIRONMENT_HOOK,
        )
        self.hook_managers[self.RESULT_HANDLERS] = ExtensionManager(
            namespace=self.RESULT_HANDLERS,
        )

        self.driver_managers = OrderedDict()
        self.driver_managers[self.TEST_DISCOVERY] = ExtensionManager(
            namespace=self.TEST_DISCOVERY,
        )
        self.driver_managers[self.TEST_RUNNER] = ExtensionManager(
            namespace=self.TEST_RUNNER,
        )

    @classmethod
    def testing_plugin_manager(cls, hook_managers, driver_managers):
        """Create a fabricated plugin manager for testing.

        """
        plugin_manager = cls.__new__(cls)
        plugin_manager.hook_managers = OrderedDict(hook_managers)
        plugin_manager.driver_managers = OrderedDict(driver_managers)
        return plugin_manager

    def _hook_extension_option_prefix(self, extension):
        name = uncamelcase(extension.name, sep='-').replace('_', '-')
        option_prefix = '--with-{0}'.format(name)
        dest_prefix = name.replace('-', '_')
        return option_prefix, dest_prefix

    def _namespace_to_option(self, namespace):
        parts = self._namespace_to_option_parts[namespace]
        option = '--{0}'.format('-'.join(parts))
        dest = '_'.join(parts)
        return option, dest

    def _add_hook_extension_arguments(self, extension, parser):
        option_prefix, dest_prefix = self._hook_extension_option_prefix(
            extension)
        extension.plugin.add_parser_arguments(
            parser, extension.name, option_prefix, dest_prefix)

    def _create_hook_plugin(self, extension, args, **kwargs):
        option_prefix, dest_prefix = self._hook_extension_option_prefix(
            extension)
        plugin = extension.plugin.from_args(
            args, extension.name, dest_prefix, **kwargs)
        if plugin is not None and plugin.enabled:
            return plugin
        return None

    def _add_driver_extension_arguments(self, extension, parser, option_prefix,
                                        dest_prefix):
        extension.plugin.add_parser_arguments(
            parser, option_prefix, dest_prefix)

    def add_plugin_arguments(self, parser):
        """Add plugin arguments to argument parser.

        Parameters
        ----------
        parser : argparse.ArgumentParser
            The main haas ArgumentParser.

        """
        for manager in self.hook_managers.values():
            if len(list(manager)) == 0:
                continue
            manager.map(self._add_hook_extension_arguments, parser)
        for namespace, manager in self.driver_managers.items():
            choices = list(sorted(manager.names()))
            if len(choices) == 0:
                continue
            option, dest = self._namespace_to_option(namespace)
            parser.add_argument(
                option, help=self._help[namespace], dest=dest,
                choices=choices, default='default')
            option_prefix = '{0}-'.format(option)
            dest_prefix = '{0}_'.format(dest)
            manager.map(self._add_driver_extension_arguments,
                        parser, option_prefix, dest_prefix)

    def get_enabled_hook_plugins(self, hook, args, **kwargs):
        """Get enabled plugins for specified hook name.

        """
        manager = self.hook_managers[hook]
        if len(list(manager)) == 0:
            return []
        return [
            plugin for plugin in manager.map(
                self._create_hook_plugin, args, **kwargs)
            if plugin is not None
        ]

    def get_driver(self, namespace, parsed_args, **kwargs):
        """Get mutually-exlusive plugin for plugin namespace.

        """
        option, dest = self._namespace_to_option(namespace)
        dest_prefix = '{0}_'.format(dest)
        driver_name = getattr(parsed_args, dest, 'default')
        driver_extension = self.driver_managers[namespace][driver_name]
        return driver_extension.plugin.from_args(
            parsed_args, dest_prefix, **kwargs)
				did_you_mean.append(query_word+query_separators[i])
				did_you_mean_markdown.append(query_word+query_separators[i])
				did_you_mean_html.append(query_word+query_separators[i])
		
		did_you_mean = "".join(did_you_mean)
		did_you_mean_markdown = "".join(did_you_mean_markdown)
		did_you_mean_html = "".join(did_you_mean_html)
		output = OrderedDict()
		output["your_input"] = query
		output["did_you_mean"] = did_you_mean
		output["did_you_mean_markdown"] = did_you_mean_markdown
		output["did_you_mean_html"] = did_you_mean_html
		
		self.stdout.write('') 
		self.stdout.write("Result:")
		for i,k in output.items():
			self.stdout.write("   '%s': %s" % (i,k.encode("utf-8",errors="replace"),))
		self.stdout.write('')
		
		return output
		pass
	
	def do_import_file(self,filepath,locale):
		with codecs.open(filepath, encoding='utf-8') as file:
			for line in file:
				sentences, separators_s = self.split_bysentences(line)
				if len([sentence for sentence in sentences if sentence]):
					for sentence in sentences:
						self.import_sentence(sentence,locale)
		pass
	
Exemple #48
0
class HasParameters(object):
    """This class provides an implementation of the IHasParameters interface."""

    _do_not_promote = [
        'get_expr_depends', 'get_referenced_compnames',
        'get_referenced_varpaths', 'get_metadata'
    ]

    def __init__(self, parent):
        self._parameters = OrderedDict()
        self._allowed_types = ['continuous']
        if obj_has_interface(parent, ISolver):
            self._allowed_types.append('unbounded')
        self._parent = None if parent is None else weakref.ref(parent)

    def __getstate__(self):
        state = self.__dict__.copy()
        state['_parent'] = self.parent
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        parent = state['_parent']
        self._parent = None if parent is None else weakref.ref(parent)

    @property
    def parent(self):
        """ The object we are a delegate of. """
        return None if self._parent is None else self._parent()

    def _item_count(self):
        """This is used by the replace function to determine if a delegate from
        the target object is 'empty' or not.  If it's empty, it's not an error
        if the replacing object doesn't have this delegate.
        """
        return len(self._parameters)

    def add_parameter(self,
                      target,
                      low=None,
                      high=None,
                      scaler=None,
                      adder=None,
                      start=None,
                      fd_step=None,
                      name=None,
                      scope=None):
        """Adds a parameter or group of parameters to the driver.

        target: string or iter of strings or Parameter
            What the driver should vary during execution. A *target* is an
            expression that can reside on the left-hand side of an assignment
            statement, so typically it will be the name of a variable or
            possibly a subscript expression indicating an entry within an array
            variable, e.g., x[3]. If an iterator of targets is given, then the
            driver will set all targets given to the same value whenever it
            varies this parameter during execution. If a Parameter instance is
            given, then that instance is copied into the driver with any other
            arguments specified, overiding the values in the given parameter.

        low: float (optional)
            Minimum allowed value of the parameter. If scaler and/or adder
            is supplied, use the transformed value here. If target is an array,
            this may also be an array, but must have the same size.

        high: float (optional)
            Maximum allowed value of the parameter. If scaler and/or adder
            is supplied, use the transformed value here. If target is an array,
            this may also be an array, but must have the same size.

        scaler: float (optional)
            Value to multiply the possibly offset parameter value by. If target
            is an array, this may also be an array, but must have the same size.

        adder: float (optional)
            Value to add to parameter prior to possible scaling. If target is
            an array, this may also be an array, but must have the same size.

        start: any (optional)
            Value to set into the target or targets of a parameter before
            starting any executions. If not given, analysis will start with
            whatever values are in the target or targets at that time. If target
            is an array, this may also be an array, but must have the same size.

        fd_step: float (optional)
            Step-size to use for finite difference calculation. If no value is
            given, the differentiator will use its own default. If target is an
            array, this may also be an array, but must have the same size.

        name: str (optional)
            Name used to refer to the parameter in place of the name of the
            variable referred to in the parameter string.
            This is sometimes useful if, for example, multiple entries in the
            same array variable are declared as parameters.

        scope: object (optional)
            The object to be used as the scope when evaluating the expression.

        If neither "low" nor "high" is specified, the min and max will
        default to the values in the metadata of the variable being
        referenced.
        """

        if isinstance(target, (ParameterBase, ParameterGroup)):
            self._parameters[target.name] = target
            target.override(low, high, scaler, adder, start, fd_step, name)
        else:
            if isinstance(target, basestring):
                names = [target]
                key = target
            else:
                names = target
                key = tuple(target)

            if name is not None:
                key = name

            dups = set(self.list_param_targets()).intersection(names)
            if len(dups) == 1:
                self.parent.raise_exception(
                    "'%s' is already a Parameter"
                    " target" % dups.pop(), ValueError)
            elif len(dups) > 1:
                self.parent.raise_exception(
                    "%s are already Parameter targets" % sorted(list(dups)),
                    ValueError)

            if key in self._parameters:
                self.parent.raise_exception("%s is already a Parameter" % key,
                                            ValueError)
            try:
                _scope = self._get_scope(scope)
                if len(names) == 1:
                    target = self._create(names[0], low, high, scaler, adder,
                                          start, fd_step, key, _scope)
                else:  # defining a ParameterGroup
                    parameters = [
                        self._create(n, low, high, scaler, adder, start,
                                     fd_step, key, _scope) for n in names
                    ]
                    types = set([p.valtypename for p in parameters])
                    if len(types) > 1:
                        raise ValueError("Can't add parameter %s because "
                                         "%s are not all of the same type" %
                                         (key, " and ".join(names)))
                    target = ParameterGroup(parameters)
                self._parameters[key] = target
            except Exception:
                self.parent.reraise_exception(info=sys.exc_info())

        self.parent.config_changed()

    def _create(self, target, low, high, scaler, adder, start, fd_step, key,
                scope):
        """ Create one Parameter or ArrayParameter. """
        try:
            expreval = ExprEvaluator(target, scope)
        except Exception as err:
            raise err.__class__("Can't add parameter: %s" % err)
        if not expreval.is_valid_assignee():
            raise ValueError("Can't add parameter: '%s' is not a"
                             " valid parameter expression" % expreval.text)
        try:
            val = expreval.evaluate()
        except Exception as err:
            val = None  # Let Parameter code sort out why.

        name = key[0] if isinstance(key, tuple) else key

        if isinstance(val, ndarray):
            return ArrayParameter(target,
                                  low=low,
                                  high=high,
                                  scaler=scaler,
                                  adder=adder,
                                  start=start,
                                  fd_step=fd_step,
                                  name=name,
                                  scope=scope,
                                  _expreval=expreval,
                                  _val=val,
                                  _allowed_types=self._allowed_types)
        else:
            return Parameter(target,
                             low=low,
                             high=high,
                             scaler=scaler,
                             adder=adder,
                             start=start,
                             fd_step=fd_step,
                             name=name,
                             scope=scope,
                             _expreval=expreval,
                             _val=val,
                             _allowed_types=self._allowed_types)

    def remove_parameter(self, name):
        """Removes the parameter with the given name."""
        param = self._parameters.get(name)
        if param:
            del self._parameters[name]
        else:
            self.parent.raise_exception(
                "Trying to remove parameter '%s' "
                "that is not in this driver." % (name, ), AttributeError)
        self.parent.config_changed()

    def config_parameters(self):
        """Reconfigure parameters from potentially changed targets."""
        for param in self._parameters.values():
            param.configure()

    def get_references(self, name):
        """Return references to component `name` in preparation for subsequent
        :meth:`restore_references` call.

        name: string
            Name of component being removed.
        """
        refs = OrderedDict()
        for pname, param in self._parameters.items():
            if name in param.get_referenced_compnames():
                refs[pname] = param
        return refs

    def remove_references(self, name):
        """Remove references to component `name`.

        name: string
            Name of component being removed.
        """
        to_remove = []
        for pname, param in self._parameters.items():
            if name in param.get_referenced_compnames():
                to_remove.append(pname)

        for pname in to_remove:
            self.remove_parameter(pname)

    def restore_references(self, refs):
        """Restore references to component `name` from `refs`.

        refs: object
            Value returned by :meth:`get_references`.
        """
        for pname, param in refs.items():
            try:
                self.add_parameter(param)
            except Exception as err:
                self.parent._logger.warning(
                    "Couldn't restore parameter '%s': %s" % (pname, str(err)))

    def list_param_targets(self):
        """Returns a list of parameter targets. Note that this
        list may contain more entries than the list of Parameter,
        ParameterGroup, and ArrayParameter objects since ParameterGroup
        instances have multiple targets.
        """
        targets = []
        for param in self._parameters.values():
            targets.extend(param.targets)
        return targets

    def list_param_group_targets(self):
        """Returns a list of tuples that contain the targets for each
        parameter group.
        """
        targets = []
        for param in self.get_parameters().values():
            targets.append(tuple(param.targets))
        return targets

    def clear_parameters(self):
        """Removes all parameters."""
        for name in self._parameters.keys():
            self.remove_parameter(name)
        self._parameters = OrderedDict()

    def get_parameters(self):
        """Returns an ordered dict of parameter objects."""
        return self._parameters

    def total_parameters(self):
        """Returns the total number of values to be set."""
        return sum([param.size for param in self._parameters.values()])

    def init_parameters(self):
        """Sets all parameters to their start value if a
        start value is given
        """
        scope = self._get_scope()
        for param in self._parameters.itervalues():
            if param.start is not None:
                param.set(param.start, scope)

    def set_parameter_by_name(self, name, value, case=None, scope=None):
Exemple #49
0
            id = row[0]
            body = row[1]
            title = row[2]
            tag_string = row[3]

            processed_body = remove_tags(remove_code(body))
            processed_tags = TAG_EXTRACTOR.findall(tag_string)

            features = OrderedDict()
            features["textFeatures"] = calculate_text_features(body, title)
            features["tagFeatures"] = calculate_tag_features(processed_tags)
            features["shallowLinguisticFeatures"] = TextStatistics(
                processed_body).calculate_shallow_text_features()

            # convert ordereddicts to list
            values = []
            for category, feature in features.items():
                for value in feature.values():
                    if isinstance(value, list):
                        values.extend(value)  # arrays
                    else:
                        values.append(value)  # single Integers, Floats
            values.append(id)

            updateData.append(values)

            if len(updateData) > FLUSH_LIMIT:
                update_trainings_features(updateData, cursor, cnx)
                updateData = []
Exemple #50
0
class Config(object):
    """ KGEN configuration parameter holder """
    def __init__(self):

        # setup config parameters
        self._attrs = OrderedDict()
        self.opt_handlers = OrderedDict()

        # KGEN operation mode
        self._attrs['check_mode'] = False

        # Fortran parameters
        self._attrs['fort'] = OrderedDict()
        self._attrs['fort']['maxlinelen'] = 132

        # logging parameters
        self._attrs['logging'] = OrderedDict()
        self._attrs['logging']['select'] = OrderedDict()

        # callsite parameters
        self._attrs['callsite'] = OrderedDict()
        self._attrs['callsite']['filepath'] = ''
        self._attrs['callsite']['span'] = (-1, -1)
        self._attrs['callsite']['namepath'] = ''
        #        self._attrs['callsite']['lineafter'] = -1

        # external tool parameters
        self._attrs['bin'] = OrderedDict()
        self._attrs['bin']['pp'] = 'cpp'
        self._attrs['bin']['cpp_flags'] = '-w -traditional'
        self._attrs['bin']['fpp_flags'] = '-w'

        # search parameters
        self._attrs['search'] = OrderedDict()
        self._attrs['search']['skip_intrinsic'] = True
        self._attrs['search']['except'] = []
        self._attrs['search']['promote_exception'] = False

        # path parameters
        self._attrs['path'] = OrderedDict()
        self._attrs['path']['outdir'] = '.'
        self._attrs['path']['state'] = 'state'
        self._attrs['path']['kernel'] = 'kernel'

        # source file parameters
        self._attrs['source'] = OrderedDict()
        self._attrs['source']['isfree'] = None
        self._attrs['source']['isstrict'] = None
        self._attrs['source']['alias'] = OrderedDict()
        self._attrs['source']['file'] = OrderedDict()

        # include parameters
        self._attrs['include'] = OrderedDict()
        self._attrs['include']['macro'] = OrderedDict()
        self._attrs['include']['path'] = ['.']
        self._attrs['include']['type'] = OrderedDict()
        self._attrs['include']['import'] = OrderedDict()
        self._attrs['include']['file'] = OrderedDict()

        # add mpi frame code in kernel driver
        self._attrs['add_mpi_frame'] = OrderedDict()
        self._attrs['add_mpi_frame']['enabled'] = False
        self._attrs['add_mpi_frame']['np'] = '2'
        self._attrs['add_mpi_frame']['mpiexec'] = 'mpiexec'

        # exclude parameters
        self._attrs['exclude'] = OrderedDict()

        # debugging parameters
        self._attrs['debug'] = OrderedDict()
        self._attrs['debug']['printvar'] = []

        # plugin parameters
        self._attrs['plugin'] = OrderedDict()
        self._attrs['plugin']['priority'] = OrderedDict()

    def apply(self, cfg):
        import optparse

        if not cfg:
            raise ProgramException('Custom configuration is not provided.')

        if hasattr(cfg, 'attrs'):
            self._attrs.update(cfg.attrs)

        # parsing arguments
        parser = optparse.OptionParser(usage=cfg.usage, version=cfg.version)

        # add default options
        parser.add_option("-s",
                          "--syntax-check",
                          dest="syntax_check",
                          action='store_true',
                          default=False,
                          help="KGEN Syntax Check Mode")
        parser.add_option("-i",
                          "--include-ini",
                          dest="include_ini",
                          action='store',
                          type='string',
                          default=None,
                          help="information used for analysis")
        parser.add_option("-e",
                          "--exclude-ini",
                          dest="exclude_ini",
                          action='store',
                          type='string',
                          default=None,
                          help="information excluded for analysis")
        parser.add_option("-I",
                          dest="include",
                          action='append',
                          type='string',
                          default=None,
                          help="include path information used for analysis")
        parser.add_option("-D",
                          dest="macro",
                          action='append',
                          type='string',
                          default=None,
                          help="macro information used for analysis")
        parser.add_option("--outdir",
                          dest="outdir",
                          action='store',
                          type='string',
                          default=None,
                          help="path to create outputs")
        parser.add_option("--source",
                          dest="source",
                          action='append',
                          type='string',
                          default=None,
                          help="Setting source file related properties")
        parser.add_option("--skip-intrinsic",
                          dest="skip_intrinsic",
                          action='store_true',
                          default=False,
                          help=optparse.SUPPRESS_HELP)
        parser.add_option("--noskip-intrinsic",
                          dest="noskip_intrinsic",
                          action='store_true',
                          default=False,
                          help=optparse.SUPPRESS_HELP)
        parser.add_option(
            "--intrinsic",
            dest="intrinsic",
            action='append',
            type='string',
            default=None,
            help=
            "Specifying resolution for intrinsic procedures during searching")
        parser.add_option("--debug",
                          dest="debug",
                          action='append',
                          type='string',
                          help=optparse.SUPPRESS_HELP)
        parser.add_option("--logging",
                          dest="logging",
                          action='append',
                          type='string',
                          help=optparse.SUPPRESS_HELP)
        parser.add_option("--add-mpi-frame",
                          dest="add_mpi_frame",
                          type='string',
                          default=None,
                          help='Add MPI frame codes in kernel_driver.')

        # add custom options
        if hasattr(cfg, 'options'):
            for opt_handler, args, kwargs in cfg.options:
                parser.add_option(*args, **kwargs)
                self.opt_handlers[kwargs['dest']] = opt_handler

        opts, args = parser.parse_args()

        if len(args) < 1:
            print 'ERROR: No call-site information is provided in command line.'
            sys.exit(-1)

        if opts.syntax_check:
            self._process_default_flags(opts)
            self._attrs['check_mode'] = args
            return

        # old options
        if opts.skip_intrinsic:
            print "skip-intrinsic flag is discarded. Please use --intrinsic skip instead"
            sys.exit(-1)
        if opts.noskip_intrinsic:
            print "noskip-intrinsic flag is discarded. Please use --intrinsic noskip instead"
            sys.exit(-1)

        callsite = args[0].split(':', 1)
        if not os.path.isfile(callsite[0]):
            print 'ERROR: %s can not be found.' % callsite[0]
            sys.exit(-1)

        # set callsite filepath
        self.callsite['filepath'] = callsite[0]

        # set namepath if exists in command line argument
        if len(callsite) == 2:
            self.callsite['namepath'] = callsite[1].lower()
        elif len(callsite) > 2:
            print 'ERROR: Unrecognized call-site information(Syntax -> filepath[:subprogramname]): %s' % str(
                callsite)
            sys.exit(-1)

        # process default flags
        self._process_default_flags(opts)

        # process custom flags
        for optname, opt_handler in self.opt_handlers.items():
            opt = getattr(opts, optname, None)
            if opt and optname in self.opt_handlers:
                self.opt_handlers[optname](opt)

    def _process_default_flags(self, opts):

        # check if exists fpp or cpp
        output = ''
        try:
            output = exec_cmd('which cpp', show_error_msg=False).strip()
        except Exception as e:
            pass
        if output.endswith('cpp'):
            self.bin['pp'] = output
        else:
            output = ''
            try:
                output = exec_cmd('which fpp', show_error_msg=False).strip()
            except Exception as e:
                pass
            if output.endswith('fpp'):
                self.bin['pp'] = output
            else:
                print 'ERROR: neither cpp or fpp is found'
                sys.exit(-1)

        # parsing intrinsic skip option
        if opts.intrinsic:
            subflags = []
            for line in opts.intrinsic:
                subflags.extend(line.split(','))

            for subf in subflags:
                if subf and subf.find('=') > 0:
                    key, value = subf.split('=')
                    if key == 'except':
                        self._attrs['search']['except'].extend(
                            value.split(';'))
                    elif key == 'add_intrinsic':
                        Intrinsic_Procedures.extend(
                            [name.lower() for name in value.split(';')])
                    else:
                        raise UserException(
                            'Unknown intrinsic sub option: %s' % subf)
                else:
                    if subf == 'skip':
                        self._attrs['search']['skip_intrinsic'] = True
                    elif subf == 'noskip':
                        self._attrs['search']['skip_intrinsic'] = False
                    else:
                        raise UserException(
                            'Unknown intrinsic option(s) in %s' % subf)

        # parsing include parameters
        if opts.include:
            for inc in opts.include:
                inc_eq = inc.split('=')
                if len(inc_eq) == 1:
                    for inc_colon in inc_eq[0].split(':'):
                        self._attrs['include']['path'].append(inc_colon)
                if len(inc_eq) == 1:
                    for inc_colon in inc_eq[0].split(':'):
                        self._attrs['include']['path'].append(inc_colon)
                elif len(inc_eq) == 2:
                    # TODO: support path for each file
                    pass
                else:
                    raise UserException('Wrong format include: %s' % inc)

        if opts.include_ini:
            process_include_option(opts.include_ini, self._attrs['include'])

        if opts.exclude_ini:
            process_exclude_option(opts.exclude_ini, self._attrs['exclude'])

        # parsing macro parameters
        if opts.macro:
            for line in opts.macro:
                for macro in line.split(','):
                    macro_eq = macro.split('=')
                    if len(macro_eq) == 1:
                        self._attrs['include']['macro'][macro_eq[0]] = '1'
                    elif len(macro_eq) == 2:
                        self._attrs['include']['macro'][
                            macro_eq[0]] = macro_eq[1]
                    else:
                        raise UserException('Wrong format include: %s' % inc)

        files = None
        if opts.source:
            for line in opts.source:
                flags = OrderedDict()
                for subflag in line.lower().split(','):
                    if subflag.find('=') > 0:
                        key, value = subflag.split('=')
                        if key == 'file':
                            flags[key] = value.split(':')
                        elif key == 'alias':
                            p1, p2 = value.split(':')
                            if p1.endswith('/'): p1 = p1[:-1]
                            if p2.endswith('/'): p2 = p2[:-1]
                            self._attrs['source']['alias'][p1] = p2
                        else:
                            flags[key] = value
                    else:
                        flags[subflag] = None

                isfree = None
                isstrict = None
                if flags.has_key('format'):
                    if flags['format'] == 'free': isfree = True
                    elif flags['format'] == 'fixed': isfree = False
                    else:
                        raise UserException(
                            'format subflag of source flag should be either free or fixed.'
                        )

                if flags.has_key('strict'):
                    if flags['strict'] == 'yes': isstrict = True
                    elif flags['strict'] == 'no': isstrict = False
                    else:
                        raise UserException(
                            'strict subflag of source flag should be either yes or no.'
                        )

                if flags.has_key('file'):
                    subflags = OrderedDict()
                    if isfree: subflags['isfree'] = isfree
                    if isstrict: subflags['isstrict'] = isstrict
                    for file in flags['file']:
                        abspath = os.path.abspath(file)
                        if files is None: files = []
                        files.append(abspath)
                        self._attrs['source']['file'][abspath] = subflags
                else:
                    if isfree: self._attrs['source']['isfree'] = isfree
                    if isstrict: self._attrs['source']['isstrict'] = isstrict

        # dupulicate paths per each alias
        if files is None:
            newpath = set()
            for path in self._attrs['include']['path']:
                newpath.add(path)
                for p1, p2 in self._attrs['source']['alias'].iteritems():
                    if path.startswith(p1):
                        newpath.add(p2 + path[len(p1):])
                    elif path.startswith(p2):
                        newpath.add(p1 + path[len(p2):])
            self._attrs['include']['path'] = list(newpath)

        newfile = OrderedDict()
        for path, value in self._attrs['include']['file'].iteritems():
            newfile[path] = value
            for p1, p2 in self._attrs['source']['alias'].iteritems():
                if path.startswith(p1):
                    newpath = p2 + path[len(p1):]
                    newfile[newpath] = deepcopy(value)
                elif path.startswith(p2):
                    newpath = p1 + path[len(p2):]
                    newfile[newpath] = deepcopy(value)
        self._attrs['include']['file'] = newfile

        for path, value in self._attrs['include']['file'].iteritems():
            if value.has_key('path'):
                newpath = set()
                for path in value['path']:
                    newpath.add(path)
                    for p1, p2 in self._attrs['source']['alias'].iteritems():
                        if path.startswith(p1):
                            newpath.add(p2 + path[len(p1):])
                        elif path.startswith(p2):
                            newpath.add(p1 + path[len(p2):])
                value['path'] = list(newpath)

        # parsing debugging options
        if opts.debug:
            for dbg in opts.debug:
                param_path, value = dbg.split('=')
                param_split = param_path.lower().split('.')
                value_split = value.lower().split(',')
                curdict = self._attrs['debug']
                for param in param_split[:-1]:
                    curdict = curdict[param]
                exec('curdict[param_split[-1]] = value_split')

        # parsing logging options
        if opts.logging:
            for log in opts.logging:
                param_path, value = log.split('=')
                param_split = param_path.lower().split('.')
                value_split = value.lower().split(',')
                curdict = self._attrs['logging']
                for param in param_split[:-1]:
                    curdict = curdict[param]
                exec('curdict[param_split[-1]] = value_split')

        # mpi frame code in kernel driver
        if opts.add_mpi_frame:
            self._attrs['add_mpi_frame']['enabled'] = True
            for checkparams in opts.add_mpi_frame.split(','):
                key, value = checkparams.split('=')
                key = key.lower()
                if key in ['np', 'mpiexec']:
                    self._attrs['add_mpi_frame'][key] = value
                else:
                    print 'WARNING: %s is not supported add_mpi_frame parameter' % key

        if opts.outdir:
            self._attrs['path']['outdir'] = opts.outdir

        # create state directories and change working directory
        if not os.path.exists(self._attrs['path']['outdir']):
            os.makedirs(self._attrs['path']['outdir'])
        os.chdir(self._attrs['path']['outdir'])

    def __getattr__(self, name):
        return self._attrs[name]
Exemple #51
0
class ConfigArgParser(argparse.ArgumentParser):
    """Drop-in replacement for argparse.ConfigArgParser that adds support for
    environment variables and .ini or .yaml-style config files.
    """

    def __init__(self,
                 prog=None,
                 usage=None,
                 description=None,
                 epilog=None,
                 version=None,
                 parents=None,
                 formatter_class=argparse.HelpFormatter,
                 prefix_chars='-',
                 fromfile_prefix_chars=None,
                 argument_default=None,
                 conflict_handler='error',
                 add_help=True,

                 add_config_file_help=True,
                 add_env_var_help=True,

                 auto_env_var_prefix=None,

                 config_file_parser=None,
                 default_config_files=None,
                 ignore_unknown_config_file_keys=False,
                 allow_unknown_config_file_keys=False,  # deprecated

                 args_for_setting_config_path=None,
                 config_arg_is_required=False,
                 config_arg_help_message="config file path",

                 args_for_writing_out_config_file=None,
                 write_out_config_file_arg_help_message="takes the current command line "
                                                        "args and writes them out to a config file at the given path, then "
                                                        "exits"
                 ):

        # noinspection PyPep8
        """Supports all the same args as the argparse.ConfigArgParser
                constructor, as well as the following additional args.
        
                Additional Args:
                    add_config_file_help: Whether to add a description of config file
                        syntax to the help message.
                    add_env_var_help: Whether to add something to the help message for
                        args that can be set through environment variables.
                    auto_env_var_prefix: If set to a string instead of None, all config-
                        file-settable options will become also settable via environment
                        variables whose names are this prefix followed by the config
                        file key, all in upper case. (eg. setting this to "foo_" will
                        allow an arg like "--my-arg" to also be set via the FOO_MY_ARG
                        environment variable)
                    config_file_parser: An instance of a parser to be used for parsing
                        config files. Default: ConfigFileParser()
                    default_config_files: When specified, this list of config files will
                        be parsed in order, with the values from each config file
                        taking precedence over previous ones. This allows an application
                        to look for config files in multiple standard locations such as
                        the install directory, home directory, and current directory:
                        ["<install dir>/app_config.ini",
                        "~/.my_app_config.ini",
                        "./app_config.txt"]
                    ignore_unknown_config_file_keys: If true, settings that are found
                        in a config file but don't correspond to any defined
                        configargparse args will be ignored. If false, they will be
                        processed and appended to the commandline like other args, and
                        can be retrieved using parse_known_args() instead of parse_args()
                    allow_unknown_config_file_keys:
                        @deprecated
                        Use ignore_unknown_config_file_keys instead.
        
                        If true, settings that are found in a config file but don't
                        correspond to any defined configargparse args, will still be
                        processed and appended to the command line (eg. for
                        parsing with parse_known_args()). If false, they will be ignored.
        
                    args_for_setting_config_path: A list of one or more command line
                        args to be used for specifying the config file path
                        (eg. ["-c", "--config-file"]). Default: []
                    config_arg_is_required: When args_for_setting_config_path is set,
                        set this to True to always require users to provide a config path.
                    config_arg_help_message: the help message to use for the
                        args listed in args_for_setting_config_path.
                    args_for_writing_out_config_file: A list of one or more command line
                        args to use for specifying a config file output path. If
                        provided, these args cause configargparse to write out a config
                        file with settings based on the other provided commandline args,
                        environment variants and defaults, and then to exit.
                        (eg. ["-w", "--write-out-config-file"]). Default: []
                    write_out_config_file_arg_help_message: The help message to use for
                        the args in args_for_writing_out_config_file.
                """

        if not parents:
            # noinspection PyPep8
            parents = list()

        if not default_config_files:
            default_config_files = list()

        if not args_for_setting_config_path:
            args_for_setting_config_path = list()

        if not args_for_writing_out_config_file:
            args_for_writing_out_config_file = list()

        self._source_to_settings = None

        self._add_config_file_help = add_config_file_help
        self._add_env_var_help = add_env_var_help
        self._auto_env_var_prefix = auto_env_var_prefix

        # extract kwargs that can be passed to the super constructor
        # noinspection PyPep8
        kwargs_for_super = dict(
            (k, v) for k, v in locals().items() if k in [
                "prog", "usage", "description", "epilog", "version",
                "parents",
                "formatter_class", "prefix_chars",
                "fromfile_prefix_chars",
                "argument_default", "conflict_handler", "add_help"])
        if sys.version_info >= (3, 3) and "version" in kwargs_for_super:
            # noinspection PyPep8
            del kwargs_for_super[
                "version"]  # version arg deprecated in v3.3

        argparse.ArgumentParser.__init__(self, **kwargs_for_super)

        # parse the additional args
        if config_file_parser is None:
            self._config_file_parser = ConfigFileParser()
        else:
            self._config_file_parser = config_file_parser
        self._default_config_files = default_config_files
        # noinspection PyPep8
        self._ignore_unknown_config_file_keys = ignore_unknown_config_file_keys \
                                                or allow_unknown_config_file_keys
        if args_for_setting_config_path:
            # noinspection PyPep8
            self.add_argument(*args_for_setting_config_path,
                              dest="config_file",
                              required=config_arg_is_required,
                              help=config_arg_help_message,
                              is_config_file_arg=True)

        if args_for_writing_out_config_file:
            # noinspection PyPep8
            self.add_argument(*args_for_writing_out_config_file,
                              dest="write_out_config_file_to_this_path",
                              metavar="CONFIG_OUTPUT_PATH",
                              help=write_out_config_file_arg_help_message,
                              is_write_out_config_file_arg=True)

    def parse_args(self, args=None, namespace=None,
                   config_file_contents=None, env_vars=os.environ):
        # noinspection PyPep8
        """Supports all the same args as the ConfigArgParser.parse_args(..),
                as well as the following additional args.
        
                Additional Args:
                    args: a list of args as in argparse, or a string (eg. "-x -y bla")
                    config_file_contents: String. Used for testing.
                    env_vars: Dictionary. Used for testing.
                """
        # noinspection PyPep8
        args, argv = self.parse_known_args(args=args,
                                           namespace=namespace,
                                           config_file_contents=config_file_contents,
                                           env_vars=env_vars)
        if argv:
            self.error('unrecognized arguments: %s' % ' '.join(argv))
        return args

    # noinspection PyPep8
    def parse_known_args(self, args=None, namespace=None,
                         config_file_contents=None,
                         env_vars=os.environ):
        # noinspection PyPep8
        """Supports all the same args as the ConfigArgParser.parse_args(..),
                as well as the following additional args.
        
                Additional Args:
                    args: a list of args as in argparse, or a string (eg. "-x -y bla")
                    config_file_contents: String. Used for testing.
                    env_vars: Dictionary. Used for testing.
                """
        if args is None:
            args = sys.argv[1:]
        elif type(args) == str:
            args = args.split()
        else:
            args = list(args)

        for a in self._actions:
            a.is_positional_arg = not a.option_strings

        # maps string describing the source (eg. env var) to a settings dict
        # to keep track of where values came from (used by print_values())
        self._source_to_settings = OrderedDict()
        if args:
            # noinspection PyPep8
            a_v_pair = (
                None, list(args))  # copy args list to isolate changes
            # noinspection PyPep8
            self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = {
                '': a_v_pair}

        # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed
        if self._auto_env_var_prefix is not None:
            for a in self._actions:
                config_file_keys = self.get_possible_config_keys(a)
                # noinspection PyPep8
                if config_file_keys and not (
                                    a.env_var or a.is_positional_arg
                            or a.is_config_file_arg or a.is_write_out_config_file_arg):
                    # noinspection PyPep8
                    stripped_config_file_key = config_file_keys[
                        0].strip(
                        self.prefix_chars)
                    # noinspection PyPep8
                    a.env_var = (self._auto_env_var_prefix +
                                 stripped_config_file_key).replace('-',
                                                                   '_').upper()

        # add env var settings to the commandline that aren't there already
        env_var_args = []
        # noinspection PyPep8
        actions_with_env_var_values = [a for a in self._actions
                                       if
                                       not a.is_positional_arg and a.env_var and a.env_var in env_vars
                                       and not already_on_command_line(
                                           args, a.option_strings)]
        for action in actions_with_env_var_values:
            key = action.env_var
            value = env_vars[key]
            env_var_args += self.convert_setting_to_command_line_arg(
                action, key, value)

        args += env_var_args

        if env_var_args:
            self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict(
                [(a.env_var, (a, env_vars[a.env_var]))
                 for a in actions_with_env_var_values])

        # prepare for reading config file(s)
        # noinspection PyPep8
        known_config_keys = dict(
            (config_key, action) for action in self._actions
            for config_key in self.get_possible_config_keys(action))

        # open the config file(s)
        if config_file_contents:
            stream = StringIO(config_file_contents)
            stream.name = "method arg"
            config_streams = [stream]
        else:
            config_streams = self._open_config_files(args)

        config_settings = OrderedDict()
        # parse each config file
        # noinspection PyPep8
        for stream in config_streams[::-1]:
            try:
                config_settings = self._config_file_parser.parse(stream)
            except ConfigFileParserException as e:
                self.error(e)
            finally:
                if hasattr(stream, "close"):
                    stream.close()

            # add each config setting to the commandline unless it's there already
            config_args = []
            for key, value in config_settings.items():
                if key in known_config_keys:
                    action = known_config_keys[key]
                    discard_this_key = already_on_command_line(
                        args, action.option_strings)
                else:
                    action = None
                    # noinspection PyPep8
                    discard_this_key = self._ignore_unknown_config_file_keys or \
                                       already_on_command_line(
                                           args,
                                           self.get_command_line_key_for_unknown_config_file_setting(
                                               key))

                if not discard_this_key:
                    # noinspection PyPep8
                    config_args += self.convert_setting_to_command_line_arg(
                        action, key, value)
                    # noinspection PyPep8
                    source_key = "%s|%s" % (
                        _CONFIG_FILE_SOURCE_KEY, stream.name)
                    if source_key not in self._source_to_settings:
                        # noinspection PyPep8
                        self._source_to_settings[
                            source_key] = OrderedDict()
                    # noinspection PyPep8
                    self._source_to_settings[source_key][key] = (
                        action, value)

            args += config_args

        # save default settings for use by print_values()
        default_settings = OrderedDict()
        for action in self._actions:
            # noinspection PyPep8
            cares_about_default_value = (not action.is_positional_arg or
                                         action.nargs in [OPTIONAL,
                                                          ZERO_OR_MORE])
            # noinspection PyPep8
            if (already_on_command_line(args, action.option_strings) or
                    not cares_about_default_value or
                        action.default is None or
                        action.default == SUPPRESS or
                        type(
                            action) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE):
                continue
            else:
                if action.option_strings:
                    key = action.option_strings[-1]
                else:
                    key = action.dest
                default_settings[key] = (action, str(action.default))

        if default_settings:
            # noinspection PyPep8
            self._source_to_settings[
                _DEFAULTS_SOURCE_KEY] = default_settings

        # parse all args (including commandline, config file, and env var)
        # noinspection PyPep8
        namespace, unknown_args = argparse.ArgumentParser.parse_known_args(
            self, args=args, namespace=namespace)

        # handle any args that have is_write_out_config_file_arg set to true
        # noinspection PyPep8
        user_write_out_config_file_arg_actions = [a for a in
                                                  self._actions
                                                  if getattr(a,
                                                             "is_write_out_config_file_arg",
                                                             False)]
        if user_write_out_config_file_arg_actions:
            output_file_paths = []
            # noinspection PyPep8
            for action in user_write_out_config_file_arg_actions:
                # check if the user specified this arg on the commandline
                output_file_path = getattr(namespace, action.dest, None)
                if output_file_path:
                    # validate the output file path
                    try:
                        # noinspection PyUnusedLocal
                        with open(output_file_path, "w") as output_file:
                            output_file_paths.append(output_file_path)
                    except IOError as e:
                        # noinspection PyPep8
                        raise ValueError(
                            "Couldn't open %s for writing: %s" % (
                                output_file_path, e))

            if output_file_paths:
                # generate the config file contents
                config_items = self.get_items_for_config_file_output(
                    self._source_to_settings, namespace)
                # noinspection PyPep8
                contents = self._config_file_parser.serialize(
                    config_items)
                for output_file_path in output_file_paths:
                    with open(output_file_path, "w") as output_file:
                        output_file.write(contents)
                if len(output_file_paths) == 1:
                    output_file_paths = output_file_paths[0]
                # noinspection PyPep8
                self.exit(message="Wrote config file to " + str(
                    output_file_paths))
        return namespace, unknown_args

    def get_command_line_key_for_unknown_config_file_setting(self, key):
        # noinspection PyPep8
        """Compute a commandline arg key to be used for a config file setting
                that doesn't correspond to any defined configargparse arg (and so
                doesn't have a user-specified commandline arg key).
        
                Args:
                    key: The config file key that was being set.
                """
        key_without_prefix_chars = key.strip(self.prefix_chars)
        # noinspection PyPep8
        command_line_key = self.prefix_chars[
                               0] * 2 + key_without_prefix_chars

        return command_line_key

    def get_items_for_config_file_output(self, source_to_settings,
                                         parsed_namespace):
        # noinspection PyPep8
        """Does the inverse of config parsing by taking parsed values and
                converting them back to a string representing config file contents.
        
                Args:
                    source_to_settings: the dictionary created within parse_known_args()
                    parsed_namespace: namespace object created within parse_known_args()
                Returns:
                    an OrderedDict with the items to be written to the config file
                """
        config_file_items = OrderedDict()
        for source, settings in source_to_settings.items():
            if source == _COMMAND_LINE_SOURCE_KEY:
                _, existing_command_line_args = settings['']
                for action in self._actions:
                    # noinspection PyPep8
                    config_file_keys = self.get_possible_config_keys(
                        action)
                    # noinspection PyPep8
                    if config_file_keys and not action.is_positional_arg and \
                            already_on_command_line(
                                existing_command_line_args,
                                action.option_strings):
                        # noinspection PyPep8
                        value = getattr(parsed_namespace, action.dest,
                                        None)
                        if value is not None:
                            if type(value) is bool:
                                value = str(value).lower()
                            elif type(value) is list:
                                # noinspection PyPep8
                                value = "[" + ", ".join(
                                    map(str, value)) + "]"
                            # noinspection PyPep8
                            config_file_items[
                                config_file_keys[0]] = value

            elif source == _ENV_VAR_SOURCE_KEY:
                for key, (action, value) in settings.items():
                    # noinspection PyPep8
                    config_file_keys = self.get_possible_config_keys(
                        action)
                    if config_file_keys:
                        # noinspection PyPep8
                        value = getattr(parsed_namespace, action.dest,
                                        None)
                        if value is not None:
                            # noinspection PyPep8
                            config_file_items[
                                config_file_keys[0]] = value
            elif source.startswith(_CONFIG_FILE_SOURCE_KEY):
                for key, (action, value) in settings.items():
                    config_file_items[key] = value
            elif source == _DEFAULTS_SOURCE_KEY:
                for key, (action, value) in settings.items():
                    # noinspection PyPep8
                    config_file_keys = self.get_possible_config_keys(
                        action)
                    if config_file_keys:
                        # noinspection PyPep8
                        value = getattr(parsed_namespace, action.dest,
                                        None)
                        if value is not None:
                            # noinspection PyPep8
                            config_file_items[
                                config_file_keys[0]] = value
        return config_file_items

    def convert_setting_to_command_line_arg(self, action, key, value):
        # noinspection PyPep8
        """Converts a config file or env var key/value to a list of
                commandline args to append to the commandline.
        
                Args:
                    action: The action corresponding to this setting, or None if this
                        is a config file setting that doesn't correspond to any
                        defined configargparse arg.
                    key: The config file key or env var name
                    value: The raw value string from the config file or env var
                """
        if type(value) != str:
            raise ValueError("type(value) != str: %s" % str(value))

        args = []
        if action is None:
            # noinspection PyPep8
            command_line_key = \
                self.get_command_line_key_for_unknown_config_file_setting(
                    key)
        else:
            command_line_key = action.option_strings[-1]

        if value.lower() == "true":
            if action is not None:
                # noinspection PyPep8
                if type(
                        action) not in ACTION_TYPES_THAT_DONT_NEED_A_VALUE:
                    # noinspection PyPep8
                    self.error(
                        "%s set to 'True' rather than a value" % key)
            # noinspection PyPep8
            args.append(command_line_key)
        elif value.startswith("[") and value.endswith("]"):
            if action is not None:
                # noinspection PyProtectedMember
                if type(action) != argparse._AppendAction:
                    # noinspection PyPep8
                    self.error(
                        ("%s can't be set to a list '%s' unless its "
                         "action type is changed to 'append'") % (
                            key, value))
            for list_elem in value[1:-1].split(","):
                # noinspection PyPep8
                args.append(command_line_key)
                # noinspection PyPep8
                args.append(list_elem.strip())
        else:
            if action is not None:
                if type(action) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE:
                    # noinspection PyPep8
                    self.error(
                        "%s is a flag but is being set to '%s'" % (
                            key, value))
            # noinspection PyPep8
            args.append(command_line_key)
            # noinspection PyPep8
            args.append(value)
        return args

    def get_possible_config_keys(self, action):
        # noinspection PyPep8
        """This method decides which actions can be set in a config file and
                what their keys will be. It returns a list of 0 or more config keys that
                can be used to set the given action's value in a config file.
                """
        keys = []
        for arg in action.option_strings:
            if any([arg.startswith(2 * c) for c in self.prefix_chars]):
                # noinspection PyPep8
                keys += [arg[2:],
                         arg]  # eg. for '--bla' return ['bla', '--bla']

        return keys

    # noinspection PyPep8
    def _open_config_files(self, command_line_args):
        # noinspection PyPep8
        """Tries to parse config file path(s) from within command_line_args. 
                Returns a list of opened config files, including files specified on the 
                commandline as well as any default_config_files specified in the
                constructor that are present on disk.
        
                Args:
                    command_line_args: List of all args (already split on spaces)
                """
        # open any default config files
        # noinspection PyPep8
        config_files = [open(f) for f in map(
            os.path.expanduser, self._default_config_files) if
                        os.path.isfile(f)]

        if not command_line_args:
            return config_files

        # list actions with is_config_file_arg=True. Its possible there is more
        # than one such arg.
        # noinspection PyPep8
        user_config_file_arg_actions = [
            a for a in self._actions if
            getattr(a, "is_config_file_arg", False)]

        if not user_config_file_arg_actions:
            return config_files

        # noinspection PyPep8
        for action in user_config_file_arg_actions:
            # try to parse out the config file path by using a clean new
            # ConfigArgParser that only knows this one arg/action.
            arg_parser = argparse.ArgumentParser(
                prefix_chars=self.prefix_chars,
                add_help=False)

            arg_parser._add_action(action)

            # make parser not exit on error by replacing its error method.
            # Otherwise it sys.exits(..) if, for example, config file 
            # is_required=True and user doesn't provide it.
            # noinspection PyUnusedLocal
            def error_method(message):
                """
                
                :param message: 
                :return: 
                """
                pass

            # noinspection PyArgumentList,PyPep8
            arg_parser.error = types.MethodType(error_method,
                                                arg_parser)

            # check whether the user provided a value 
            # noinspection PyPep8
            parsed_arg = arg_parser.parse_known_args(
                args=command_line_args)
            if not parsed_arg:
                continue
            namespace, _ = parsed_arg
            user_config_file = getattr(namespace, action.dest, None)
            if not user_config_file:
                continue
            # validate the user-provided config file path
            user_config_file = os.path.expanduser(user_config_file)
            if not os.path.isfile(user_config_file):
                self.error('File not found: %s' % user_config_file)

            config_files += [open(user_config_file)]

        return config_files

    def format_values(self):
        """Returns a string with all args and settings and where they came from
        (eg. commandline, config file, environment variable or default)
        """
        source_key_to_display_value_map = {
            _COMMAND_LINE_SOURCE_KEY: "Command Line Args: ",
            _ENV_VAR_SOURCE_KEY: "Environment Variables:\n",
            _CONFIG_FILE_SOURCE_KEY: "Config File (%s):\n",
            _DEFAULTS_SOURCE_KEY: "Defaults:\n"
        }

        r = StringIO()
        for source, settings in self._source_to_settings.items():
            source = source.split("|")
            # noinspection PyPep8
            source = source_key_to_display_value_map[source[0]] % tuple(
                source[1:])
            r.write(source)
            for key, (action, value) in settings.items():
                if key:
                    r.write("  %-19s%s\n" % (key + ":", value))
                else:
                    if type(value) is str:
                        r.write("  %s\n" % value)
                    elif type(value) is list:
                        r.write("  %s\n" % ' '.join(value))

        return r.getvalue()

    def print_values(self, file=sys.stdout):
        # noinspection PyPep8
        """Prints the format_values() string (to sys.stdout or another file)."""
        file.write(self.format_values())

    def format_help(self):
        """
        
        :return: 
        """
        msg = ""
        added_config_file_help = False
        added_env_var_help = False
        if self._add_config_file_help:
            default_config_files = self._default_config_files
            cc = 2 * self.prefix_chars[0]  # eg. --
            # noinspection PyPep8
            config_settable_args = [(arg, a) for a in self._actions for
                                    arg in
                                    a.option_strings if
                                    self.get_possible_config_keys(
                                        a) and not
                                    (
                                        a.dest == "help" or a.is_config_file_arg or
                                        a.is_write_out_config_file_arg)]
            # noinspection PyPep8
            config_path_actions = [a for a in
                                   self._actions if
                                   getattr(a, "is_config_file_arg",
                                           False)]

            if config_settable_args and (default_config_files or
                                             config_path_actions):
                # noinspection PyPep8
                self._add_config_file_help = False  # prevent duplication
                added_config_file_help = True

                # noinspection PyPep8
                msg += (
                           "Args that start with '%s' (eg. %s) can also be set in "
                           "a config file") % (
                           cc, config_settable_args[0][0])
                # noinspection PyPep8
                config_arg_string = " or ".join(a.option_strings[0]
                                                for a in
                                                config_path_actions if
                                                a.option_strings)
                if config_arg_string:
                    # noinspection PyPep8
                    config_arg_string = "specified via " + config_arg_string
                if default_config_files or config_arg_string:
                    # noinspection PyPep8
                    msg += " (%s)." % " or ".join(default_config_files +
                                                  [config_arg_string])
                # noinspection PyPep8
                msg += " " + self._config_file_parser.get_syntax_description()

        if self._add_env_var_help:
            env_var_actions = [(a.env_var, a) for a in self._actions
                               if getattr(a, "env_var", None)]
            for env_var, a in env_var_actions:
                env_var_help_string = "   [env var: %s]" % env_var
                if not a.help:
                    a.help = ""
                if env_var_help_string not in a.help:
                    a.help += env_var_help_string
                    added_env_var_help = True
                    # noinspection PyPep8
                    self._add_env_var_help = False  # prevent duplication

        if added_env_var_help or added_config_file_help:
            value_sources = ["defaults"]
            if added_config_file_help:
                value_sources = ["config file values"] + value_sources
            if added_env_var_help:
                # noinspection PyPep8
                value_sources = [
                                    "environment variables"] + value_sources
            # noinspection PyPep8
            msg += (
                       " If an arg is specified in more than one place, then "
                       "commandline values override %s.") % (
                       " which override ".join(value_sources))
        if msg:
            self.description = (self.description or "") + " " + msg

        return argparse.ArgumentParser.format_help(self)
Exemple #52
0
    def parse_known_args(self, args=None, namespace=None,
                         config_file_contents=None,
                         env_vars=os.environ):
        # noinspection PyPep8
        """Supports all the same args as the ConfigArgParser.parse_args(..),
                as well as the following additional args.
        
                Additional Args:
                    args: a list of args as in argparse, or a string (eg. "-x -y bla")
                    config_file_contents: String. Used for testing.
                    env_vars: Dictionary. Used for testing.
                """
        if args is None:
            args = sys.argv[1:]
        elif type(args) == str:
            args = args.split()
        else:
            args = list(args)

        for a in self._actions:
            a.is_positional_arg = not a.option_strings

        # maps string describing the source (eg. env var) to a settings dict
        # to keep track of where values came from (used by print_values())
        self._source_to_settings = OrderedDict()
        if args:
            # noinspection PyPep8
            a_v_pair = (
                None, list(args))  # copy args list to isolate changes
            # noinspection PyPep8
            self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = {
                '': a_v_pair}

        # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed
        if self._auto_env_var_prefix is not None:
            for a in self._actions:
                config_file_keys = self.get_possible_config_keys(a)
                # noinspection PyPep8
                if config_file_keys and not (
                                    a.env_var or a.is_positional_arg
                            or a.is_config_file_arg or a.is_write_out_config_file_arg):
                    # noinspection PyPep8
                    stripped_config_file_key = config_file_keys[
                        0].strip(
                        self.prefix_chars)
                    # noinspection PyPep8
                    a.env_var = (self._auto_env_var_prefix +
                                 stripped_config_file_key).replace('-',
                                                                   '_').upper()

        # add env var settings to the commandline that aren't there already
        env_var_args = []
        # noinspection PyPep8
        actions_with_env_var_values = [a for a in self._actions
                                       if
                                       not a.is_positional_arg and a.env_var and a.env_var in env_vars
                                       and not already_on_command_line(
                                           args, a.option_strings)]
        for action in actions_with_env_var_values:
            key = action.env_var
            value = env_vars[key]
            env_var_args += self.convert_setting_to_command_line_arg(
                action, key, value)

        args += env_var_args

        if env_var_args:
            self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict(
                [(a.env_var, (a, env_vars[a.env_var]))
                 for a in actions_with_env_var_values])

        # prepare for reading config file(s)
        # noinspection PyPep8
        known_config_keys = dict(
            (config_key, action) for action in self._actions
            for config_key in self.get_possible_config_keys(action))

        # open the config file(s)
        if config_file_contents:
            stream = StringIO(config_file_contents)
            stream.name = "method arg"
            config_streams = [stream]
        else:
            config_streams = self._open_config_files(args)

        config_settings = OrderedDict()
        # parse each config file
        # noinspection PyPep8
        for stream in config_streams[::-1]:
            try:
                config_settings = self._config_file_parser.parse(stream)
            except ConfigFileParserException as e:
                self.error(e)
            finally:
                if hasattr(stream, "close"):
                    stream.close()

            # add each config setting to the commandline unless it's there already
            config_args = []
            for key, value in config_settings.items():
                if key in known_config_keys:
                    action = known_config_keys[key]
                    discard_this_key = already_on_command_line(
                        args, action.option_strings)
                else:
                    action = None
                    # noinspection PyPep8
                    discard_this_key = self._ignore_unknown_config_file_keys or \
                                       already_on_command_line(
                                           args,
                                           self.get_command_line_key_for_unknown_config_file_setting(
                                               key))

                if not discard_this_key:
                    # noinspection PyPep8
                    config_args += self.convert_setting_to_command_line_arg(
                        action, key, value)
                    # noinspection PyPep8
                    source_key = "%s|%s" % (
                        _CONFIG_FILE_SOURCE_KEY, stream.name)
                    if source_key not in self._source_to_settings:
                        # noinspection PyPep8
                        self._source_to_settings[
                            source_key] = OrderedDict()
                    # noinspection PyPep8
                    self._source_to_settings[source_key][key] = (
                        action, value)

            args += config_args

        # save default settings for use by print_values()
        default_settings = OrderedDict()
        for action in self._actions:
            # noinspection PyPep8
            cares_about_default_value = (not action.is_positional_arg or
                                         action.nargs in [OPTIONAL,
                                                          ZERO_OR_MORE])
            # noinspection PyPep8
            if (already_on_command_line(args, action.option_strings) or
                    not cares_about_default_value or
                        action.default is None or
                        action.default == SUPPRESS or
                        type(
                            action) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE):
                continue
            else:
                if action.option_strings:
                    key = action.option_strings[-1]
                else:
                    key = action.dest
                default_settings[key] = (action, str(action.default))

        if default_settings:
            # noinspection PyPep8
            self._source_to_settings[
                _DEFAULTS_SOURCE_KEY] = default_settings

        # parse all args (including commandline, config file, and env var)
        # noinspection PyPep8
        namespace, unknown_args = argparse.ArgumentParser.parse_known_args(
            self, args=args, namespace=namespace)

        # handle any args that have is_write_out_config_file_arg set to true
        # noinspection PyPep8
        user_write_out_config_file_arg_actions = [a for a in
                                                  self._actions
                                                  if getattr(a,
                                                             "is_write_out_config_file_arg",
                                                             False)]
        if user_write_out_config_file_arg_actions:
            output_file_paths = []
            # noinspection PyPep8
            for action in user_write_out_config_file_arg_actions:
                # check if the user specified this arg on the commandline
                output_file_path = getattr(namespace, action.dest, None)
                if output_file_path:
                    # validate the output file path
                    try:
                        # noinspection PyUnusedLocal
                        with open(output_file_path, "w") as output_file:
                            output_file_paths.append(output_file_path)
                    except IOError as e:
                        # noinspection PyPep8
                        raise ValueError(
                            "Couldn't open %s for writing: %s" % (
                                output_file_path, e))

            if output_file_paths:
                # generate the config file contents
                config_items = self.get_items_for_config_file_output(
                    self._source_to_settings, namespace)
                # noinspection PyPep8
                contents = self._config_file_parser.serialize(
                    config_items)
                for output_file_path in output_file_paths:
                    with open(output_file_path, "w") as output_file:
                        output_file.write(contents)
                if len(output_file_paths) == 1:
                    output_file_paths = output_file_paths[0]
                # noinspection PyPep8
                self.exit(message="Wrote config file to " + str(
                    output_file_paths))
        return namespace, unknown_args
Exemple #53
0
class ArgumentParser(argparse.ArgumentParser):
    """Drop-in replacement for argparse.ArgumentParser that adds support for
    environment variables and .ini or .yaml-style config files.
    """
    def __init__(
            self,
            add_config_file_help=True,
            add_env_var_help=True,
            auto_env_var_prefix=None,
            default_config_files=[],
            ignore_unknown_config_file_keys=False,
            config_file_parser_class=DefaultConfigFileParser,
            args_for_setting_config_path=[],
            config_arg_is_required=False,
            config_arg_help_message="config file path",
            args_for_writing_out_config_file=[],
            write_out_config_file_arg_help_message="takes the current command line "
        "args and writes them out to a config file at the given path, then "
        "exits",
            **kwargs):
        """Supports args of the argparse.ArgumentParser constructor
        as **kwargs, as well as the following additional args.

        Additional Args:
            add_config_file_help: Whether to add a description of config file
                syntax to the help message.
            add_env_var_help: Whether to add something to the help message for
                args that can be set through environment variables.
            auto_env_var_prefix: If set to a string instead of None, all config-
                file-settable options will become also settable via environment
                variables whose names are this prefix followed by the config
                file key, all in upper case. (eg. setting this to "foo_" will
                allow an arg like "--my-arg" to also be set via the FOO_MY_ARG
                environment variable)
            default_config_files: When specified, this list of config files will
                be parsed in order, with the values from each config file
                taking precedence over pervious ones. This allows an application
                to look for config files in multiple standard locations such as
                the install directory, home directory, and current directory.
                Also, shell * syntax can be used to specify all conf files in a
                directory. For exmaple:
                ["/etc/conf/app_config.ini",
                 "/etc/conf/conf-enabled/*.ini",
                "~/.my_app_config.ini",
                "./app_config.txt"]
            ignore_unknown_config_file_keys: If true, settings that are found
                in a config file but don't correspond to any defined
                configargparse args will be ignored. If false, they will be
                processed and appended to the commandline like other args, and
                can be retrieved using parse_known_args() instead of parse_args()
            config_file_parser_class: configargparse.ConfigFileParser subclass
                which determines the config file format. configargparse comes
                with DefaultConfigFileParser and YAMLConfigFileParser.
            args_for_setting_config_path: A list of one or more command line
                args to be used for specifying the config file path
                (eg. ["-c", "--config-file"]). Default: []
            config_arg_is_required: When args_for_setting_config_path is set,
                set this to True to always require users to provide a config path.
            config_arg_help_message: the help message to use for the
                args listed in args_for_setting_config_path.
            args_for_writing_out_config_file: A list of one or more command line
                args to use for specifying a config file output path. If
                provided, these args cause configargparse to write out a config
                file with settings based on the other provided commandline args,
                environment variants and defaults, and then to exit.
                (eg. ["-w", "--write-out-config-file"]). Default: []
            write_out_config_file_arg_help_message: The help message to use for
                the args in args_for_writing_out_config_file.
        """
        self._add_config_file_help = add_config_file_help
        self._add_env_var_help = add_env_var_help
        self._auto_env_var_prefix = auto_env_var_prefix

        argparse.ArgumentParser.__init__(self, **kwargs)

        # parse the additional args
        if config_file_parser_class is None:
            self._config_file_parser = DefaultConfigFileParser()
        else:
            self._config_file_parser = config_file_parser_class()

        self._default_config_files = default_config_files
        self._ignore_unknown_config_file_keys = ignore_unknown_config_file_keys
        if args_for_setting_config_path:
            self.add_argument(*args_for_setting_config_path,
                              dest="config_file",
                              required=config_arg_is_required,
                              help=config_arg_help_message,
                              is_config_file_arg=True)

        if args_for_writing_out_config_file:
            self.add_argument(*args_for_writing_out_config_file,
                              dest="write_out_config_file_to_this_path",
                              metavar="CONFIG_OUTPUT_PATH",
                              help=write_out_config_file_arg_help_message,
                              is_write_out_config_file_arg=True)

    def parse_args(self,
                   args=None,
                   namespace=None,
                   config_file_contents=None,
                   env_vars=os.environ):
        """Supports all the same args as the ArgumentParser.parse_args(..),
        as well as the following additional args.

        Additional Args:
            args: a list of args as in argparse, or a string (eg. "-x -y bla")
            config_file_contents: String. Used for testing.
            env_vars: Dictionary. Used for testing.
        """
        args, argv = self.parse_known_args(
            args=args,
            namespace=namespace,
            config_file_contents=config_file_contents,
            env_vars=env_vars)
        if argv:
            self.error('unrecognized arguments: %s' % ' '.join(argv))
        return args

    def parse_known_args(self,
                         args=None,
                         namespace=None,
                         config_file_contents=None,
                         env_vars=os.environ):
        """Supports all the same args as the ArgumentParser.parse_args(..),
        as well as the following additional args.

        Additional Args:
            args: a list of args as in argparse, or a string (eg. "-x -y bla")
            config_file_contents: String. Used for testing.
            env_vars: Dictionary. Used for testing.
        """
        if args is None:
            args = sys.argv[1:]
        elif isinstance(args, str):
            args = args.split()
        else:
            args = list(args)

        # normalize args by converting args like --key=value to --key value
        normalized_args = list()
        for arg in args:
            if arg and arg[0] in self.prefix_chars and '=' in arg:
                key, value = arg.split('=', 1)
                normalized_args.append(key)
                normalized_args.append(value)
            else:
                normalized_args.append(arg)
        args = normalized_args

        for a in self._actions:
            a.is_positional_arg = not a.option_strings

        # maps a string describing the source (eg. env var) to a settings dict
        # to keep track of where values came from (used by print_values()).
        # The settings dicts for env vars and config files will then map
        # the config key to an (argparse Action obj, string value) 2-tuple.
        self._source_to_settings = OrderedDict()
        if args:
            a_v_pair = (None, list(args))  # copy args list to isolate changes
            self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = {'': a_v_pair}

        # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed
        if self._auto_env_var_prefix is not None:
            for a in self._actions:
                config_file_keys = self.get_possible_config_keys(a)
                if config_file_keys and not (
                        a.env_var or a.is_positional_arg or
                        a.is_config_file_arg or a.is_write_out_config_file_arg
                        or isinstance(a, argparse._HelpAction)):
                    stripped_config_file_key = config_file_keys[0].strip(
                        self.prefix_chars)
                    a.env_var = (self._auto_env_var_prefix +
                                 stripped_config_file_key).replace(
                                     '-', '_').upper()

        # add env var settings to the commandline that aren't there already
        env_var_args = []
        actions_with_env_var_values = [
            a for a in self._actions
            if not a.is_positional_arg and a.env_var and a.env_var in env_vars
            and not already_on_command_line(args, a.option_strings)
        ]
        for action in actions_with_env_var_values:
            key = action.env_var
            value = env_vars[key]
            # Make list-string into list.
            if action.nargs or isinstance(action, argparse._AppendAction):
                element_capture = re.match('\[(.*)\]', value)
                if element_capture:
                    value = [
                        val.strip()
                        for val in element_capture.group(1).split(',')
                        if val.strip()
                    ]
            env_var_args += self.convert_item_to_command_line_arg(
                action, key, value)

        args = args + env_var_args

        if env_var_args:
            self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict([
                (a.env_var, (a, env_vars[a.env_var]))
                for a in actions_with_env_var_values
            ])

        # before parsing any config files, check if -h was specified.
        supports_help_arg = any(a for a in self._actions
                                if isinstance(a, argparse._HelpAction))
        skip_config_file_parsing = supports_help_arg and ("-h" in args
                                                          or "--help" in args)

        # prepare for reading config file(s)
        known_config_keys = dict(
            (config_key, action) for action in self._actions
            for config_key in self.get_possible_config_keys(action))

        # open the config file(s)
        config_streams = []
        if config_file_contents is not None:
            stream = StringIO(config_file_contents)
            stream.name = "method arg"
            config_streams = [stream]
        elif not skip_config_file_parsing:
            config_streams = self._open_config_files(args)

        # parse each config file
        for stream in reversed(config_streams):
            try:
                config_items = self._config_file_parser.parse(stream)
            except ConfigFileParserException as e:
                self.error(e)
            finally:
                if hasattr(stream, "close"):
                    stream.close()

            # add each config item to the commandline unless it's there already
            config_args = []
            for key, value in config_items.items():
                if key in known_config_keys:
                    action = known_config_keys[key]
                    discard_this_key = already_on_command_line(
                        args, action.option_strings)
                else:
                    action = None
                    discard_this_key = self._ignore_unknown_config_file_keys or \
                        already_on_command_line(
                            args,
                            [self.get_command_line_key_for_unknown_config_file_setting(key)])

                if not discard_this_key:
                    config_args += self.convert_item_to_command_line_arg(
                        action, key, value)
                    source_key = "%s|%s" % (_CONFIG_FILE_SOURCE_KEY,
                                            stream.name)
                    if source_key not in self._source_to_settings:
                        self._source_to_settings[source_key] = OrderedDict()
                    self._source_to_settings[source_key][key] = (action, value)

            args = args + config_args

        # save default settings for use by print_values()
        default_settings = OrderedDict()
        for action in self._actions:
            cares_about_default_value = (not action.is_positional_arg
                                         or action.nargs
                                         in [OPTIONAL, ZERO_OR_MORE])
            if (already_on_command_line(args, action.option_strings)
                    or not cares_about_default_value or action.default is None
                    or action.default == SUPPRESS or isinstance(
                        action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE)):
                continue
            else:
                if action.option_strings:
                    key = action.option_strings[-1]
                else:
                    key = action.dest
                default_settings[key] = (action, str(action.default))

        if default_settings:
            self._source_to_settings[_DEFAULTS_SOURCE_KEY] = default_settings

        # parse all args (including commandline, config file, and env var)
        namespace, unknown_args = argparse.ArgumentParser.parse_known_args(
            self, args=args, namespace=namespace)
        # handle any args that have is_write_out_config_file_arg set to true
        # check if the user specified this arg on the commandline
        output_file_paths = [
            getattr(namespace, a.dest, None) for a in self._actions
            if getattr(a, "is_write_out_config_file_arg", False)
        ]
        output_file_paths = [a for a in output_file_paths if a is not None]
        self.write_config_file(namespace, output_file_paths, exit_after=True)
        return namespace, unknown_args

    def write_config_file(self,
                          parsed_namespace,
                          output_file_paths,
                          exit_after=False):
        """Write the given settings to output files.

        Args:
            parsed_namespace: namespace object created within parse_known_args()
            output_file_paths: any number of file paths to write the config to
            exit_after: whether to exit the program after writing the config files
        """
        for output_file_path in output_file_paths:
            # validate the output file path
            try:
                with open(output_file_path, "w") as output_file:
                    pass
            except IOError as e:
                raise ValueError("Couldn't open %s for writing: %s" %
                                 (output_file_path, e))
        if output_file_paths:
            # generate the config file contents
            config_items = self.get_items_for_config_file_output(
                self._source_to_settings, parsed_namespace)
            file_contents = self._config_file_parser.serialize(config_items)
            for output_file_path in output_file_paths:
                with open(output_file_path, "w") as output_file:
                    output_file.write(file_contents)
            message = "Wrote config file to " + ", ".join(output_file_paths)
            if exit_after:
                self.exit(0, message)
            else:
                print(message)

    def get_command_line_key_for_unknown_config_file_setting(self, key):
        """Compute a commandline arg key to be used for a config file setting
        that doesn't correspond to any defined configargparse arg (and so
        doesn't have a user-specified commandline arg key).

        Args:
            key: The config file key that was being set.
        """
        key_without_prefix_chars = key.strip(self.prefix_chars)
        command_line_key = self.prefix_chars[0] * 2 + key_without_prefix_chars

        return command_line_key

    def get_items_for_config_file_output(self, source_to_settings,
                                         parsed_namespace):
        """Converts the given settings back to a dictionary that can be passed
        to ConfigFormatParser.serialize(..).

        Args:
            source_to_settings: the dictionary described in parse_known_args()
            parsed_namespace: namespace object created within parse_known_args()
        Returns:
            an OrderedDict where keys are strings and values are either strings
            or lists
        """
        config_file_items = OrderedDict()
        for source, settings in source_to_settings.items():
            if source == _COMMAND_LINE_SOURCE_KEY:
                _, existing_command_line_args = settings['']
                for action in self._actions:
                    config_file_keys = self.get_possible_config_keys(action)
                    if config_file_keys and not action.is_positional_arg and \
                        already_on_command_line(existing_command_line_args,
                                                action.option_strings):
                        value = getattr(parsed_namespace, action.dest, None)
                        if value is not None:
                            if isinstance(value, bool):
                                value = str(value).lower()
                            elif callable(action.type):
                                found = [
                                    i for i in range(
                                        0,
                                        len(existing_command_line_args) - 1)
                                    if existing_command_line_args[i] in
                                    config_file_keys
                                ]
                                if found:
                                    value = existing_command_line_args[
                                        found[-1] + 1]
                            config_file_items[config_file_keys[0]] = value

            elif source == _ENV_VAR_SOURCE_KEY:
                for key, (action, value) in settings.items():
                    config_file_keys = self.get_possible_config_keys(action)
                    if config_file_keys:
                        value = getattr(parsed_namespace, action.dest, None)
                        if value is not None:
                            config_file_items[config_file_keys[0]] = value
            elif source.startswith(_CONFIG_FILE_SOURCE_KEY):
                for key, (action, value) in settings.items():
                    config_file_items[key] = value
            elif source == _DEFAULTS_SOURCE_KEY:
                for key, (action, value) in settings.items():
                    config_file_keys = self.get_possible_config_keys(action)
                    if config_file_keys:
                        value = getattr(parsed_namespace, action.dest, None)
                        if value is not None:
                            config_file_items[config_file_keys[0]] = value
        return config_file_items

    def convert_item_to_command_line_arg(self, action, key, value):
        """Converts a config file or env var key + value to a list of
        commandline args to append to the commandline.

        Args:
            action: The argparse Action object for this setting, or None if this
                config file setting doesn't correspond to any defined
                configargparse arg.
            key: string (config file key or env var name)
            value: parsed value of type string or list
        """
        args = []

        if action is None:
            command_line_key = \
                self.get_command_line_key_for_unknown_config_file_setting(key)
        else:
            command_line_key = action.option_strings[-1]

        # handle boolean value
        if action is not None and isinstance(
                action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE):
            if value.lower() in ("true", "yes", "1"):
                args.append(command_line_key)
            elif value.lower() in ("false", "no", "0"):
                # don't append when set to "false" / "no"
                pass
            else:
                self.error("Unexpected value for %s: '%s'. Expecting 'true', "
                           "'false', 'yes', 'no', '1' or '0'" % (key, value))
        elif isinstance(value, list):
            if action is None or isinstance(action, argparse._AppendAction):
                for list_elem in value:
                    args.append(command_line_key)
                    args.append(str(list_elem))
            elif (isinstance(action, argparse._StoreAction)
                  and action.nargs in ('+', '*')) or (isinstance(
                      action.nargs, int) and action.nargs > 1):
                args.append(command_line_key)
                for list_elem in value:
                    args.append(str(list_elem))
            else:
                self.error((
                    "%s can't be set to a list '%s' unless its action type is changed "
                    "to 'append' or nargs is set to '*', '+', or > 1") %
                           (key, value))
        elif isinstance(value, str):
            args.append(command_line_key)
            args.append(value)
        else:
            raise ValueError("Unexpected value type %s for value: %s" %
                             (type(value), value))

        return args

    def get_possible_config_keys(self, action):
        """This method decides which actions can be set in a config file and
        what their keys will be. It returns a list of 0 or more config keys that
        can be used to set the given action's value in a config file.
        """
        keys = []

        # Do not write out the config options for writing out a config file
        if getattr(action, 'is_write_out_config_file_arg', None):
            return keys

        for arg in action.option_strings:
            if any([arg.startswith(2 * c) for c in self.prefix_chars]):
                keys += [arg[2:],
                         arg]  # eg. for '--bla' return ['bla', '--bla']

        return keys

    def _open_config_files(self, command_line_args):
        """Tries to parse config file path(s) from within command_line_args.
        Returns a list of opened config files, including files specified on the
        commandline as well as any default_config_files specified in the
        constructor that are present on disk.

        Args:
            command_line_args: List of all args (already split on spaces)
        """
        # open any default config files
        config_files = [
            open(f) for files in map(
                glob.glob, map(os.path.expanduser, self._default_config_files))
            for f in files
        ]

        # list actions with is_config_file_arg=True. Its possible there is more
        # than one such arg.
        user_config_file_arg_actions = [
            a for a in self._actions if getattr(a, "is_config_file_arg", False)
        ]

        if not user_config_file_arg_actions:
            return config_files

        for action in user_config_file_arg_actions:
            # try to parse out the config file path by using a clean new
            # ArgumentParser that only knows this one arg/action.
            arg_parser = argparse.ArgumentParser(
                prefix_chars=self.prefix_chars, add_help=False)

            arg_parser._add_action(action)

            # make parser not exit on error by replacing its error method.
            # Otherwise it sys.exits(..) if, for example, config file
            # is_required=True and user doesn't provide it.
            def error_method(self, message):
                pass

            arg_parser.error = types.MethodType(error_method, arg_parser)

            # check whether the user provided a value
            parsed_arg = arg_parser.parse_known_args(args=command_line_args)
            if not parsed_arg:
                continue
            namespace, _ = parsed_arg
            user_config_file = getattr(namespace, action.dest, None)

            if not user_config_file:
                continue
            # validate the user-provided config file path
            user_config_file = os.path.expanduser(user_config_file)
            if not os.path.isfile(user_config_file):
                self.error('File not found: %s' % user_config_file)

            config_files += [open(user_config_file)]

        return config_files

    def format_values(self):
        """Returns a string with all args and settings and where they came from
        (eg. commandline, config file, enviroment variable or default)
        """
        source_key_to_display_value_map = {
            _COMMAND_LINE_SOURCE_KEY: "Command Line Args: ",
            _ENV_VAR_SOURCE_KEY: "Environment Variables:\n",
            _CONFIG_FILE_SOURCE_KEY: "Config File (%s):\n",
            _DEFAULTS_SOURCE_KEY: "Defaults:\n"
        }

        r = StringIO()
        for source, settings in self._source_to_settings.items():
            source = source.split("|")
            source = source_key_to_display_value_map[source[0]] % tuple(
                source[1:])
            r.write(source)
            for key, (action, value) in settings.items():
                if key:
                    r.write("  %-19s%s\n" % (key + ":", value))
                else:
                    if isinstance(value, str):
                        r.write("  %s\n" % value)
                    elif isinstance(value, list):
                        r.write("  %s\n" % ' '.join(value))

        return r.getvalue()

    def print_values(self, file=sys.stdout):
        """Prints the format_values() string (to sys.stdout or another file)."""
        file.write(self.format_values())

    def format_help(self):
        msg = ""
        added_config_file_help = False
        added_env_var_help = False
        if self._add_config_file_help:
            default_config_files = self._default_config_files
            cc = 2 * self.prefix_chars[0]  # eg. --
            config_settable_args = [
                (arg, a) for a in self._actions for arg in a.option_strings
                if self.get_possible_config_keys(a)
                and not (a.dest == "help" or a.is_config_file_arg
                         or a.is_write_out_config_file_arg)
            ]
            config_path_actions = [
                a for a in self._actions
                if getattr(a, "is_config_file_arg", False)
            ]

            if config_settable_args and (default_config_files
                                         or config_path_actions):
                self._add_config_file_help = False  # prevent duplication
                added_config_file_help = True

                msg += (
                    "Args that start with '%s' (eg. %s) can also be set in "
                    "a config file") % (cc, config_settable_args[0][0])
                config_arg_string = " or ".join(a.option_strings[0]
                                                for a in config_path_actions
                                                if a.option_strings)
                if config_arg_string:
                    config_arg_string = "specified via " + config_arg_string
                if default_config_files or config_arg_string:
                    msg += " (%s)." % " or ".join(
                        tuple(default_config_files) +
                        tuple(filter(None, [config_arg_string])))
                msg += " " + self._config_file_parser.get_syntax_description()

        if self._add_env_var_help:
            env_var_actions = [(a.env_var, a) for a in self._actions
                               if getattr(a, "env_var", None)]
            for env_var, a in env_var_actions:
                env_var_help_string = "   [env var: %s]" % env_var
                if not a.help:
                    a.help = ""
                if env_var_help_string not in a.help:
                    a.help += env_var_help_string
                    added_env_var_help = True
                    self._add_env_var_help = False  # prevent duplication

        if added_env_var_help or added_config_file_help:
            value_sources = ["defaults"]
            if added_config_file_help:
                value_sources = ["config file values"] + value_sources
            if added_env_var_help:
                value_sources = ["environment variables"] + value_sources
            msg += (" If an arg is specified in more than one place, then "
                    "commandline values override %s.") % (
                        " which override ".join(value_sources))
        if msg:
            self.description = (self.description or "") + " " + msg

        return argparse.ArgumentParser.format_help(self)
    ('chr4', (0, 1351857)),
    ('chrX', (0, 22422827)),
    ('chr2LHet', (0, 368872)),
    ('chr2RHet', (0, 3288761)),
    ('chr3LHet', (0, 2555491)),
    ('chr3RHet', (0, 2517507)),
    ('chrM', (0, 19517)),
    ('chrU', (0, 10049037)),
    ('chrUextra', (0, 29004656)),
    ('chrXHet', (0, 204112)),
    ('chrYHet', (0, 347038)),
))

# No chrUextra or chrM
dm3.default = OrderedDict()
for chrom, size in dm3.items():
    if chrom in ['chrUextra', 'chrM']:
        continue
    dm3.default[chrom] = size

# No chrU*, chr*Het, or chrM
dm3.euchromatic = OrderedDict()
for chrom, size in dm3.default.items():
    if 'chrU' in chrom:
        continue
    if 'Het' in chrom:
        continue
    dm3.euchromatic[chrom] = size

mm9 = OrderedDict((
    ('chr1', (0, 197195432)),
def GenInteractions_int(
        G_system,
        g_bond_pattern,
        typepattern_to_coefftypes,
        canonical_order,  # function to sort atoms and bonds
        atomtypes_int2str,
        bondtypes_int2str,
        report_progress=False,  # print messages to sys.stderr?
        check_undefined_atomids_str=None):
    """
    GenInteractions() automatically determines a list of interactions
    present in a system of bonded atoms (argument "G_system"),
    which satisfy the bond topology present in "g_bond_pattern", and
    satisfy the atom and bond type requirements in "typepattern_to_coefftypes".

    Whenever a set of atoms in "G_system" are bonded together in a way which
    matches "g_bond_pattern", and when the atom and bond types is consistent
    with one of the entries in "typepattern_to_coefftypes", the corresponding
    list of atoms from G_system is appended to the list of results.

    These results (the list of lists of atoms participating in an interaction)
    are organized according their corresponding "coefftype", a string
    which identifies the type of interaction they obey as explained above.
    results are returned as a dictionary using "coefftype" as the lookup key.

    Arguments:

     -- typepattern_to_coefftypes is a list of 2-tuples --
    The first element of the 2-tuple is the "typepattern".
    It contains a string describing a list of atom types and bond types.
    The typepattern is associated with a "coefftype",
    which is the second element of the 2-tuple.  This is a string
    which identifies the type of interaction between the atoms.
    Later on, this string can be used to lookup the force field
    parameters for this interaction elsewhere.)

     -- Arguments: G_system, g_bond_pattern, atomtypes_int2str, bondtypes_int2str --

    G_system stores a list of atoms and bonds, and their attributes in
    "Ugraph" format.  In this format:
    Atom ID numbers are represented by indices into the G_system.verts[] list.
    Bond ID numbers are represented by indices into the G_system.edges[] list.
    Atom types are represented as integers in the G_system.verts[i].attr list.
    Bond types are represented as integers in the G_system.edges[i].attr list.
    They are converted into strings using
    atomtypes_int2str, and bondtypes_int2str.

    g_bond_pattern is a graph which specifies the type of bonding between
    the atoms required for a match. It is in Ugraph format (however the
    atom and bond types are left blank.)

    Atom and bond types are supplied by the user in string format. (These
    strings typically encode integers, but could be any string in principle.)
    The string-version of the ith atom type is stored in
       atomtypes_int2str[ G_system.verts[i].attr ]
    The string-version of the ith bond type is stored in
       bondtypes_int2str[ G_system.edges[i].attr ]

     -- The "canonical_order" argument: --

    The search for atoms with a given bond pattern often yields
    redundant matches.  There is no difference for example
    between the angle formed between three consecutively
    bonded atoms (named, 1, 2, 3, for example), and the
    angle between the same atoms in reverse order (3, 2, 1).
    However both triplets of atoms will be returned by the subgraph-
    matching algorithm when searching for ALL 3-body interactions.)

    To eliminate this redundancy, the caller must supply a "canonical_order"
    argument.  This is a function which sorts the atoms and bonds in a way
    which is consistent with the type of N-body interaction being considered.
    The atoms (and bonds) in a candidate match are rearranged by the
    canonical_order().  Then the re-ordered list of atom and bond ids is
    tested against the list of atom/bond ids in the matches-found-so-far,
    before it is added.

    """

    if report_progress:
        startatomid = 0
        sys.stderr.write('  searching for matching bond patterns:\n')
        sys.stderr.write('    0%')

    # Figure out which atoms from "G_system" bond together in a way which
    # matches the "g_bond_pattern" argument.  Organize these matches by
    # atom and bond types and store all of the non-redundant ones in
    # the "interactions_by_type" variable.

    gm = GraphMatcher(G_system, g_bond_pattern)

    interactions_by_type = defaultdict(list)

    for atombondids in gm.Matches():
        # "atombondids" is a tuple.
        #  atombondids[0] has atomIDs from G_system corresponding to g_bond_pattern
        #     (These atomID numbers are indices into the G_system.verts[] list.)
        #  atombondids[1] has bondIDs from G_system corresponding to g_bond_pattern
        #     (These bondID numbers are indices into the G_system.edges[] list.)

        # It's convenient to organize the list of interactions-between-
        # atoms in a dictionary indexed by atomtypes and bondtypes.
        # (Because many atoms and bonds typically share the same type,
        #  organizing the results this way makes it faster to check
        #  whether a given interaction matches a "typepattern" defined
        #  by the user.  We only have to check once for the whole group.)

        atombondtypes = \
            (tuple([G_system.GetVert(Iv).attr for Iv in atombondids[0]]),
             tuple([G_system.GetEdge(Ie).attr for Ie in atombondids[1]]))

        interactions_by_type[atombondtypes].append(atombondids)

        if report_progress:
            # GraphMatcher.Matches() searches for matches in an order
            # that selects a different atomid number from G_system,
            # starting at 0, and continuing up to the number of atoms (-1)
            # in the system (G_system.nv-1), and using this as the first
            # atom in the match (ie match[0][0]). This number can be used
            # to guess much progress has been made so far.
            oldatomid = startatomid
            startatomid = atombondids[0][0]
            percent_complete = (100 * startatomid) // G_system.GetNumVerts()
            # report less often as more progress made
            if percent_complete <= 4:
                old_pc = (100 * oldatomid) // G_system.GetNumVerts()
                if percent_complete > old_pc:
                    sys.stderr.write('  ' + str(percent_complete) + '%')
            elif percent_complete <= 8:
                pc_d2 = (100 * startatomid) // (2 * G_system.GetNumVerts())
                oldpc_d2 = (100 * oldatomid) // (2 * G_system.GetNumVerts())
                if pc_d2 > oldpc_d2:
                    sys.stderr.write('  ' + str(percent_complete) + '%')
            elif percent_complete <= 20:
                pc_d4 = (100 * startatomid) // (4 * G_system.GetNumVerts())
                oldpc_d4 = (100 * oldatomid) // (4 * G_system.GetNumVerts())
                if pc_d4 > oldpc_d4:
                    sys.stderr.write('  ' + str(percent_complete) + '%')
            else:
                pc_d10 = (100 * startatomid) // (10 * G_system.GetNumVerts())
                oldpc_d10 = (100 * oldatomid) // (10 * G_system.GetNumVerts())
                if pc_d10 > oldpc_d10:
                    sys.stderr.write('  ' + str(percent_complete) + '%')

    if report_progress:
        sys.stderr.write('  100%\n')
        #sys.stderr.write('    ...done\n')
        #sys.stderr.write('    Looking up available atom and bond types...')

    #coefftype_to_atomids = defaultdict(list)
    #abids_to_coefftypes = defaultdict(list)
    coefftype_to_atomids = OrderedDict()
    abids_to_coefftypes = OrderedDict()

    # -------------------- reporting progress -----------------------
    if report_progress:
        # The next interval of code is not technically necessary, but it makes
        # the printed output easier to read by excluding irrelevant interactions
        # Now, test each match to see if the atoms and bonds involved match
        # any of the type-patterns in the "typepattern_to_coefftypes" argument.

        types_atoms_all_str = set([])
        types_bonds_all_str = set([])
        for typepattern, coefftype in typepattern_to_coefftypes:
            for atombondtypes, abidslist in interactions_by_type.items():
                for Iv in atombondtypes[0]:
                    types_atoms_all_str.add(atomtypes_int2str[Iv])
                for Ie in atombondtypes[1]:
                    types_bonds_all_str.add(bondtypes_int2str[Ie])
    # ------------------ reporting progress (end) -------------------

    # ------------------ check to make sure all interactions are defined ------
    if check_undefined_atomids_str:
        # Checking for missing interactions is a headache.
        # Please excuse the messy code below.
        atomids_matched = OrderedDict()
        # Then loop through all the interactions (tuples of atoms) found by
        # GraphMatcher, sort the atoms and store them in dictionary
        # (atomids_matched) which keeps track of which interactions have
        # been defined (ie have force-field parameters assigned to them).
        # Initialize them to False, and update as interactions are found.
        for atombondtypes, abidslist in interactions_by_type.items():
            for abids in abidslist:
                abids = canonical_order(abids)
                atomids_int = tuple(abids[0])

                # NOTE TO SELF:
                # If in the future, different interactions (type_patterns) have
                # different symmetries, and canonical_order() varies from
                # interaction to interaction, then DONT loop over type_pattern:
                # for type_pattern, coefftype in typepattern_to_coefftypes)
                #     abids = canonical_order(abids, type_pattern)
                # Why:  When checking for undefined interactions,
                # we just want to make sure that SOME kind of interaction
                # involving these atoms exists.  The gruesome details of
                # force-field symmetry should not enter into this.
                # (We certainly don't want to require that different
                # interactions are simultaneously present for the same set of
                # atoms for ALL the possible different atom orderings for the
                # different possible symmetries in the force field you are using
                # Perhaps, in the future I should just use something like this:
                # atomids_int = abids[0]
                # atomids_int.sort()
                # atomids_int = tuple(atomids_int)
                # This would work for most molecules.
                # I suppose that in some some bizarre molecules containing
                # triangular or square cycles, for example, this would not
                # distinguish all 3 angles in the triangle, for example.
                # mistakenly thinking there was only one interaction there.
                # But these cases are rare.)

                if not atomids_int in atomids_matched:
                    atomids_matched[atomids_int] = False
                    # (Later on, we'll set some of these to True)

    # ------------------ check to make sure all interactions are defined (end)

    count = 0

    for typepattern, coefftype in typepattern_to_coefftypes:

        # ------------------ reporting progress -----------------------
        # The next interval of code is not technically necessary, but it makes
        # the printed output easier to read by excluding irrelevant
        # interactions

        if report_progress:

            # Check to see if the atoms or bonds referred to in typepattern
            # are (potentially) satisfied by any of the atoms present in the system.
            # If any of the required atoms for this typepattern are not present
            # in this system, then skip to the next typepattern.
            atoms_available_Iv = [
                False for Iv in range(0, g_bond_pattern.GetNumVerts())
            ]
            for Iv in range(0, g_bond_pattern.GetNumVerts()):
                for type_atom_str in types_atoms_all_str:
                    if MatchesPattern(type_atom_str, typepattern[Iv]):
                        atoms_available_Iv[Iv] = True
            atoms_available = True
            for Iv in range(0, g_bond_pattern.GetNumVerts()):
                if not atoms_available_Iv[Iv]:
                    atoms_available = False

            bonds_available_Ie = [
                False for Ie in range(0, g_bond_pattern.GetNumEdges())
            ]
            for Ie in range(0, g_bond_pattern.GetNumEdges()):
                for type_bond_str in types_bonds_all_str:
                    if MatchesPattern(
                            type_bond_str,
                            typepattern[g_bond_pattern.GetNumVerts() + Ie]):
                        bonds_available_Ie[Ie] = True
            bonds_available = True
            for Ie in range(0, g_bond_pattern.GetNumEdges()):
                if not bonds_available_Ie[Ie]:
                    bonds_available = False

            if atoms_available and bonds_available:

                # Explanation:
                # (Again) only if ALL of the atoms and bond requirements for
                # this type pattern are satisfied by at least SOME of the atoms
                # present in the this system, ...THEN print a status message.
                # (Because for complex all-atom force-fields, the number of
                # possible atom types, and typepatterns far exceeds the number
                # of atom types typically present in the system.  Otherwise
                # hundreds of kB of irrelevant information can be printed.)

                sys.stderr.write('    checking ' + coefftype +
                                 ' type requirements:'
                                 #' (atom-types,bond-types) '
                                 '\n     ' + str(typepattern) + '\n')

        # ------------------ reporting progress (end) -------------------

        for atombondtypes, abidslist in interactions_by_type.items():
            # express atom & bond types in a tuple of the original string
            # format
            types_atoms = [atomtypes_int2str[Iv] for Iv in atombondtypes[0]]
            types_bonds = [bondtypes_int2str[Ie] for Ie in atombondtypes[1]]
            type_strings = types_atoms + types_bonds

            # use string comparisons to check for a match with typepattern
            if MatchesAll(type_strings, typepattern):  # <-see "ttree_lex.py"
                for abids in abidslist:
                    # Re-order the atoms (and bonds) in a "canonical" way.
                    # Only add new interactions to the list after re-ordering
                    # them and checking that they have not been added earlier.
                    # (...well not when using the same coefftype at least.
                    #  This prevents the same triplet of atoms from
                    #  being used to calculate the bond-angle twice:
                    #  once for 1-2-3 and 3-2-1, for example.)
                    abids = canonical_order(abids)
                    redundant = False
                    if abids in abids_to_coefftypes:
                        coefftypes = abids_to_coefftypes[abids]
                        if coefftype in coefftypes:
                            redundant = True

                    if check_undefined_atomids_str:
                        atomids_int = tuple(abids[0])
                        atomids_matched[atomids_int] = True

                    if not redundant:

                        # (It's too bad python does not
                        #  have an Ordered defaultdict)
                        if coefftype in coefftype_to_atomids:
                            coefftype_to_atomids[coefftype].append(abids[0])
                        else:
                            coefftype_to_atomids[coefftype] = [abids[0]]
                        if abids in abids_to_coefftypes:
                            abids_to_coefftypes[abids].append(coefftype)
                        else:
                            abids_to_coefftypes[abids] = [coefftype]
                        count += 1

    if report_progress:
        sys.stderr.write('  (found ' + str(count) +
                         ' non-redundant matches)\n')

    if check_undefined_atomids_str:
        for atomids_int, found_match in atomids_matched.items():
            if not found_match:
                atomids_str = [
                    check_undefined_atomids_str[Iv] for Iv in atomids_int
                ]
                raise InputError(
                    'Error: A bonded interaction should exist between atoms:\n'
                    + '       ' + (',\n       '.join(atomids_str)) + '\n' +
                    '       ...however no interaction between these types of atoms has been defined\n'
                    +
                    '       This usually means that at least one of your atom TYPES is incorrect.\n'
                    +
                    '       If this is not the case, then you can override this error message by\n'
                    +
                    '       invoking moltemplate.sh without the \"-checkff\" argument.\n'
                )

    return coefftype_to_atomids
Exemple #56
0
class TextDisplayResponder(plugins.Responder):
    def __init__(self, optionMap, *args):
        plugins.Responder.__init__(self)
        self.enableColor = optionMap.has_key("zen")
        self.enableSummary = optionMap.has_key("b") and not optionMap.has_key(
            "s") and not optionMap.has_key("coll")
        self.failedTests = []
        self.resultSummary = OrderedDict()
        self.resultSummary["Tests Run"] = 0

    def getSummaryKey(self, category):
        if category == "success":
            return ""
        elif category == "bug":
            return "Known Bugs"
        elif category.startswith("faster") or category.startswith(
                "slower") or category == "smaller" or category == "larger":
            return "Performance Differences"
        elif category in ["killed", "unrunnable", "cancelled", "abandoned"]:
            return "Incomplete"
        else:
            return "Failures"

    def shouldDescribe(self, test):
        return test.state.hasFailed()

    def writeDescription(self, test, summary):
        if self.enableColor and test.state.hasFailed():
            self.printTestWithColorEnabled(test, colorer.RED, summary)
        else:
            self.describe(test, summary)

    def notifyComplete(self, test):
        if self.enableSummary:
            self.resultSummary["Tests Run"] += 1
            summaryKey = self.getSummaryKey(test.state.category)
            if summaryKey:
                self.failedTests.append(test)
                if summaryKey not in self.resultSummary:
                    self.resultSummary[summaryKey] = 0
                self.resultSummary[summaryKey] += 1

        if self.shouldDescribe(test):
            self.writeDescription(test, summary=False)

    def notifyAllComplete(self):
        if self.enableSummary:
            plugins.log.info("Results:")
            plugins.log.info("")
            if len(self.failedTests):
                plugins.log.info("Tests that did not succeed:")
                for test in self.failedTests:
                    self.writeDescription(test, summary=True)
                plugins.log.info("")
            parts = [
                summaryKey + ": " + str(count)
                for summaryKey, count in self.resultSummary.items()
            ]
            plugins.log.info(", ".join(parts))

    def printTestWithColorEnabled(self, test, color, summary):
        colorer.enableOutputColor(color)
        self.describe(test, summary)
        colorer.disableOutputColor()

    def getPrefix(self, test):
        return test.getIndent()

    def describe(self, test, summary):
        prefix = "  " if summary else self.getPrefix(test)
        desc = test.state.description()
        if summary and "\n" in desc:
            desc = " ".join(desc.splitlines()[:2])
        plugins.log.info(prefix + repr(test) + " " + desc)