def test_update(self): self.assertRaises(TypeError, OrderedDict().update, [('a', 1), ('b', 2)], None) # too many args pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] od = OrderedDict() od.update(dict(pairs)) self.assertEqual(sorted(od.items()), pairs) # dict input od = OrderedDict() od.update(**dict(pairs)) self.assertEqual(sorted(od.items()), pairs) # kwds input od = OrderedDict() od.update(pairs) self.assertEqual(list(od.items()), pairs) # pairs input od = OrderedDict() od.update([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5) self.assertEqual(list(od.items()), pairs) # mixed input # Make sure that direct calls to update do not clear previous contents # add that updates items are not moved to the end d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)]) d.update([('e', 5), ('f', 6)], g=7, d=4) self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)])
def split_sets(split, response_sets, survey_cache, split_entities = False): result = [] if split: for ds in split: result.append((ds, flatten_response_queryset(response_sets.filter(data_series=ds), survey_cache))) else: split = [None] result.append((None, flatten_response_queryset(response_sets, survey_cache))) if split_entities: result_dict = OrderedDict() series = [] for ds, qs in result: series.append(ds) for ds, qs in result: filter_dict = OrderedDict() for rs in qs: #Assign all responsesets to the entity that owns them filter_dict.setdefault(rs.entity, []).append(rs) for entity, data in filter_dict.items(): if entity not in result_dict.keys(): for s in series: # Ensure all dataseries are present in entity output result_dict.setdefault(entity, OrderedDict())[s] = [] result_dict[entity][ds] = data for entity, data in result_dict.items(): result_dict[entity] = data.items() result = result_dict return result
def run(self): results = OrderedDict() for key, qs in self.get_responsesets().items(): if isinstance(qs, list): if len(qs) == 0: results[key] = plugins.Vector([]) continue results[key] = self.scorecard.get_values(qs) if isinstance(qs,dict): result = [] for entity, data in qs.items(): result.append((entity, self.scorecard.get_values(data))) results[key] = plugins.Vector(result) if self.reverse_axis: if self.aggregate_on and self.aggregate_by_entity: for entity, data in results.items(): flipped = OrderedDict() for key, scorecard in data.get_values(): for operation, values in scorecard: flipped[operation] = flipped.get(operation, []) flipped[operation].append((key, values)) results[entity] = plugins.Vector(flipped.items()) else: flipped = OrderedDict() for key, scorecard in results.items(): for operation, values in scorecard: flipped[operation] = flipped.get(operation, []) flipped[operation].append((key, values)) results = flipped return results
def test_setdefault(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) pair_order = list(od.items()) self.assertEqual(od.setdefault('a', 10), 3) # make sure order didn't change self.assertEqual(list(od.items()), pair_order) self.assertEqual(od.setdefault('x', 10), 10) # make sure 'x' is added to the end self.assertEqual(list(od.items())[-1], ('x', 10))
def test_delitem(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) del od['a'] self.assert_('a' not in od) self.assertRaises(KeyError, od.__delitem__, 'a') self.assertEqual(list(od.items()), pairs[:2] + pairs[3:])
def update_metadata(self, new_metadata): if not new_metadata: return has_changes = False for key, val in new_metadata.items(): if self.metadata.get(key) != val: has_changes = True if not has_changes: return data = deepcopy(self.metadata) data.update(new_metadata) data = OrderedDict(sorted(data.items(), key=lambda t: t[0])) new_json = json.dumps(data, indent=4) org_content = _read_unicode_file_dammit(self.full_local_path) def replacer(m): parts = m.group(0).split('\n') return parts[0].strip() + '\n' + new_json + '\n' + parts[-1].strip() new_content = self._json_comment_re.sub(replacer, org_content) if new_content == org_content: return logger.debug("Writing new metadata") f = codecs.open(self.full_local_path, 'w', 'utf-8') f.write(new_content) f.close()
def test_init(self): self.assertRaises(TypeError, OrderedDict, ([('a', 1), ('b', 2)], None)) # too many args pairs = [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] self.assertEqual(sorted(OrderedDict(dict(pairs)).items()), pairs) # dict input self.assertEqual(sorted(OrderedDict(**dict(pairs)).items()), pairs) # kwds input self.assertEqual(list(OrderedDict(pairs).items()), pairs) # pairs input self.assertEqual( list( OrderedDict([('a', 1), ('b', 2), ('c', 9), ('d', 4)], c=3, e=5).items()), pairs) # mixed input # make sure no positional args conflict with possible kwdargs self.assertEqual( inspect.getargspec(OrderedDict.__dict__['__init__'])[0], ['self']) # Make sure that direct calls to __init__ do not clear previous contents d = OrderedDict([('a', 1), ('b', 2), ('c', 3), ('d', 44), ('e', 55)]) d.__init__([('e', 5), ('f', 6)], g=7, d=4) self.assertEqual(list(d.items()), [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5), ('f', 6), ('g', 7)])
def get(self, row, columns=None, column_start=None, super_column=None, column_finish=None, column_count=100): try: if super_column is None: data_columns = self.data[row] else: data_columns = self.data[row][super_column] results = OrderedDict() count = 0 if columns is not None: for c in columns: results[c] = data_columns[c] else: for c in sorted(data_columns.keys()): if count > column_count: break if column_start and cmp(c,column_start) < 0: continue if column_finish and cmp(c, column_finish) > 0: break results[c] = data_columns[c] count += 1 if not len(results): raise NotFoundException for key, value in results.items(): if isinstance(value, dict) and len(value) == 0: del(results[key]) if value is None: del(results[key]) return results except KeyError: raise NotFoundException
def update_metadata(self, new_metadata): if not new_metadata: return print "Update metadata?" has_changes = False for key, val in new_metadata.items(): if self.metadata.get(key) != val: has_changes = True if not has_changes: return print "Has changes" data = deepcopy(self.metadata) data.update(new_metadata) data = OrderedDict(sorted(data.items(), key=lambda t: t[0])) new_json = json.dumps(data, indent=4) f = codecs.open(self.full_local_path, 'r', 'utf-8') org_content = f.read() f.close() def replacer(m): parts = m.group(0).split('\n') return parts[0].strip() + '\n' + new_json + '\n' + parts[-1].strip() new_content = self._json_comment_re.sub(replacer, org_content) print new_content if new_content == org_content: return print "Write new content" f = codecs.open(self.full_local_path, 'w', 'utf-8') f.write(new_content) f.close()
class ProcessTerminationMonitor(plugins.Observable): def __init__(self): plugins.Observable.__init__(self) self.processesForKill = OrderedDict() self.exitHandlers = OrderedDict() def listRunningProcesses(self): processesToCheck = guiConfig.getCompositeValue("query_kill_processes", "", modeDependent=True) if "all" in processesToCheck: processesToCheck = [".*"] if len(processesToCheck) == 0: return [] running = [] triggerGroup = plugins.TextTriggerGroup(processesToCheck) for process, description in self.processesForKill.values(): if triggerGroup.stringContainsText(description): running.append("PID " + str(process.pid) + " : " + description) return running def getProcessIdentifier(self, process): # Unfortunately the child_watch_add method needs different ways to # identify the process on different platforms... if os.name == "posix": return process.pid else: return process._handle def startProcess( self, cmdArgs, description="", killOnTermination=True, exitHandler=None, exitHandlerArgs=(), **kwargs ): process = subprocess.Popen(cmdArgs, stdin=open(os.devnull), **kwargs) pidOrHandle = self.getProcessIdentifier(process) self.exitHandlers[int(pidOrHandle)] = (exitHandler, exitHandlerArgs) if killOnTermination: self.processesForKill[int(pidOrHandle)] = (process, description) gobject.child_watch_add(pidOrHandle, self.processExited) def processExited(self, pid, *args): if self.processesForKill.has_key(pid): del self.processesForKill[pid] if self.exitHandlers.has_key(pid): exitHandler, exitHandlerArgs = self.exitHandlers.pop(pid) if exitHandler: exitHandler(*exitHandlerArgs) def notifyKillProcesses(self, sig=None): # Don't leak processes if len(self.processesForKill) == 0: return diag = logging.getLogger("kill processes") self.notify("Status", "Terminating all external viewers ...") for pid, (process, description) in self.processesForKill.items(): if self.exitHandlers.has_key(pid): self.exitHandlers.pop(pid) # don't call exit handlers in this case, we're terminating self.notify("ActionProgress") diag.info("Killing '" + description + "' interactive process") killSubProcessAndChildren(process, sig)
def update_metadata(self, new_metadata): if not new_metadata: return has_changes = False for key, val in new_metadata.items(): if self.metadata.get(key) != val: has_changes = True if not has_changes: return data = deepcopy(self.metadata) data.update(new_metadata) data = OrderedDict(sorted(data.items(), key=lambda t: t[0])) new_json = json.dumps(data, indent=4) org_content = _read_unicode_file_dammit(self.full_local_path) def replacer(m): parts = m.group(0).split('\n') return parts[0].strip() + '\n' + new_json + '\n' + parts[-1].strip( ) new_content = self._json_comment_re.sub(replacer, org_content) if new_content == org_content: return logger.debug("Writing new metadata") f = codecs.open(self.full_local_path, 'w', 'utf-8') f.write(new_content) f.close()
def get_field_names_for_question_set(questions, respondent_set): fields = OrderedDict() for qu in questions: field_names = CsvFieldGenerator.get_field_names_for_question(qu, respondent_set) if field_names is not None: fields = OrderedDict(fields.items() + field_names.items()) return fields
def __new__(cls, name, bases, attrs): attrs['prefix'] = attrs.get('prefix', 'form') attrs = OrderedDict( sorted([(k, v) for k, v in attrs.items()], cmp=lambda x, y: cmp(getattr(x[1], 'order_counter', None), getattr(y[1], 'order_counter', None)))) return validators.DeclarativeMeta.__new__(cls, name, bases, attrs)
def test_form(req): # get_var_info = {'len': len(args)} var_info = OrderedDict(( ('form_keys', req.form.keys()), ('form_values', ', '.join(tuple(uni(req.form.getvalue(key)) for key in req.form.keys()))), ('form_getfirst', '%s,%s' % (req.form.getfirst('pname'), req.form.getfirst('px'))), ('form_getlist', '%s,%s' % (req.form.getlist('pname'), req.form.getlist('px'))), ('', ''), ('args_keys', req.args.keys()), ('args_values', ', '.join(tuple(uni(req.args[key]) for key in req.args.keys()))), ('args_getfirst', '%s,%s' % (req.args.getfirst('gname'), req.args.getfirst('gx'))), ('args_getlist', '%s,%s' % (req.args.getlist('gname'), req.args.getlist('gx'))), )) buff = get_header("HTTP Form args test") + \ (get_crumbnav(req), "<h2>Get Form</h2>", '<form method="get">', '<input type="text" name="gname" value="Ondřej"><br/>', '<input type="text" name="gsurname" value="Tůma"><br/>', '<input type="text" name="gx" value="1">' '<input type="text" name="gx" value="2">' '<input type="text" name="gx" value="3"><br/>', '<input type="submit" name="btn" value="Send">' '</form>', "<h2>Post Form</h2>", '<form method="post">', '<input type="text" name="pname" value="Ondřej"><br/>', '<input type="text" name="psurname" value="Tůma"><br/>', '<input type="text" name="px" value="8">' '<input type="text" name="px" value="7">' '<input type="text" name="px" value="6"><br/>', '<input type="submit" name="btn" value="Send">' '</form>', "<h2>Variables</h2>", "<table>") + \ tuple("<tr><td>%s:</td><td>%s</td></tr>" % (key, html(val)) for key, val in var_info.items()) + \ ("</table>", "<h2>Browser Headers</h2>", "<table>") + \ tuple("<tr><td>%s:</td><td>%s</td></tr>" % (key, val) for key, val in req.headers_in.items()) + \ ("</table>", "<h2>Request Variables </h2>", "<table>") + \ tuple("<tr><td>%s:</td><td>%s</td></tr>" % (key, val) for key, val in get_variables(req)) + \ ("</table>",) + \ get_footer() for line in buff: req.write(line + '\n') return state.OK
def get_book_number(phrase): """ Find book number of requested book of the Bible. Used to handle multiple forms of writing a certain Bible book. """ book_names = OrderedDict([('genesis',1) , ('gen',1) , ('gn',1), ('bereshit', 1), ('exodus',2), ('exod',2), ('ex',2), ('shemot',2), ('leviticus',3), ('lev',3), ('lv',3), ('vayikra',3), ('numbers',4), ('num',4), ('nm',4), ('bemidbar',4), ('deuteronomy',5), ('deut',5), ('dt',5), ('devarim',5), ('joshua',6), ('josh',6), ('yehoshua',6), ('judges',7), ('judg',7), ('jgs',7), ('shoftim',7), ('ruth',8), ('ru',8), ('1 samuel',9), ('1samuel',9), ('1 sam',9), ('1sam',9), ('1 sm',9), ('1sm',9), ('1 shmuel',9), ('1shmuel',9), ('2 samuel',10), ('2samuel',10), ('2 sam',10), ('2sam',10), ('2 sm',10), ('2sm',10), ('2 shmuel',10), ('2shmuel',10), ('1 kings',11), ('1kings',11), ('1 kgs',11), ('1kgs',11), ('1 melachim',11), ('1melachim',11), ('2 kings',12), ('2kings',12), ('2 kgs',12), ('2kgs',12), ('2 melachim',12), ('2melachim',12), ('1 chronicles',13), ('1chronicles',13), ('1 chron',13), ('1chron',13), ('1 chr',13), ('1chr',13), ('2 chronicles',14), ('2chronicles',14), ('2 chron',14), ('2chron',14), ('2 chr',14), ('2chr',14), ('ezra',15), ('ezr',15), ('nehemiah',16), ('neh',16), ('esther',17), ('est',17), ('job',18), ('jb',18), ('psalms',19), ('psalm',19), ('ps',19), ('pss',19), ('proverbs',20), ('prov',20), ('prv',20), ('ecclesiastes',21), ('eccles',21), ('eccl',21), ('song of solomon',22), ('song of songs',22), ('song of sol',22), ('sg',22), ('isaiah',23), ('isa',23), ('yeshayahu',23), ('yeshaya',23), ('jeremiah',24), ('jer',24), ('yirmiyahu',24), ('yirmiyah',24), ('lamentations',25), ('lam',25), ('ezekiel',26), ('ezek',26), ('yechezkel',26), ('daniel',27), ('dan',27), ('dn',27), ('hosea',28), ('hos',28), ('hoshea',28), ('joel',29), ('jl',29), ('yoel',29), ('amos',30), ('am',30), ('obadiah',31), ('obad',31), ('ob',31), ('ovadiah',31), ('ovadyah',31), ('jonah',32), ('jon',32), ('micah',33), ('mic',33), ('michah',33), ('nahum',34), ('nah',34), ('na',34), ('nachum',34), ('habakkuk',35), ('hab',35), ('hb',35), ('chavakuk',35), ('zephaniah',36), ('zeph',36), ('zep',36), ('tzefaniah',36), ('tzefanyah',36), ('haggai',37), ('hag',37), ('hg',37), ('chaggai',37), ('zechariah',38), ('zech',38), ('zec',38), ('zecharya',38), ('zecharyah',38), ('zechariyah',38), ('malachi',39), ('mal',39), ('matthew',40), ('mathew',40), ('matt',40), ('mat',40), ('mt',40), ('mark',41), ('mk',41), ('luke',42), ('lk',42), ('john',43), ('jn',43), ('acts',44), ('acts of the apostles',44), ('romans',45), ('rom',45), ('1 corinthians',46), ('1corinthians',46), ('1 cor',46), ('1cor',46), ('2 corinthians',47), ('2corinthians',47), ('2 cor',47), ('2cor',47), ('galatians',48), ('gal',48), ('ephesians',49), ('eph',49), ('philippians',50), ('phil',50), ('colossians',51), ('col',51), ('1 thessalonians',52), ('1thessalonians',52), ('1 thess',52), ('1thess',52), ('1 thes',52), ('1thes',52), ('2 thessalonians',53), ('2thessalonians',53), ('2 thess',53), ('2thess',53), ('2 thes',53), ('2thes',53), ('1 timothy',54), ('1timothy',54), ('1st timothy',54), ('1 tim',54), ('1tim',54), ('1 tm',54), ('1tm',54), ('2 timothy', 55), ('2timothy',55), ('2nd timothy',55), ('2 tim',55), ('2tim',55), ('2 tm',55), ('2tm',55), ('titus',56), ('ti',56), ('philemon',57), ('philem',57), ('phlm',57), ('hebrews',58), ('heb',58), ('james',59), ('jas',59), ('1 peter',60), ('1peter',60), ('1 pet',60), ('1pet',60), ('1 pt',60), ('1pt',60), ('2 peter',61), ('2peter',61), ('2 pet',61), ('2pet',61), ('2 pt',61), ('2pt',61), ('1 john',62), ('1john',62), ('1 jn',62), ('1jn',62), ('2 john',63), ('2john',63), ('2 jn',63), ('2jn',63), ('3 john',64), ('3john',64), ('3 jn',64), ('3jn',64), ('jude',65), ('revelation',66), ('revelations',66), ('rev',66), ('rv',66), ('judith',67), ('judeth',67), ('jdt',67), ('wisdom',68), ('wis',68), ('wisdom of solomon', 68), ('tobit',69), ('tob',69), ('sirach',70), ('sir',70), ('ecclesiasticus',70), ('baruch',71), ('bar',71), ('1 maccabees',72), ('1maccabees',72), ('1 macc',72), ('1macc',72), ('1 mac',72), ('1mac', 72), ('2 maccabees',73), ('2maccabees',73), ('2 macc', 73), ('2macc',73), ('2 mac',73), ('2mac',73), ('3 maccabees',74), ('3maccabees',74), ('3 macc',74), ('3macc',74), ('3 mac',74), ('3mac',74), ('4 maccabees',75), ('4maccabees',75), ('4 macc',75), ('4macc',75), ('4 mac',75), ('4mac',75), ('rest of daniel',76), ('additions to daniel',76), ('adddan',76), ('song of the three children',76), ('prayer of azariah',76), ('rest of esther',77), ('additions to esther',77), ('addesth',77), ('prayer of manasses',78), ('prayer of manasseh',78), ('manasses',78), ('manasseh',78), ('prman',78), ('1 esdras',79), ('1esdras',79), ('1 esd',79), ('1esd',79), ('2 esdras',80), ('2esdras',80), ('2 esd',80), ('2esd',80), ('story of susanna',81), ('susanna',81), ('sus',81), ('bel and the dragon',82), ('bel',82)]) sorted_books = OrderedDict(sorted(book_names.items(), key=lambda t: len(t[0]), reverse=True)) for key, value in sorted_books.items(): if search(r'\b' + key + r'\b', phrase): return value return False
def get_book_number(phrase): """ Find book number of requested book of the Bible. Used to handle multiple forms of writing a certain Bible book. """ book_names = OrderedDict([('genesis',1) , ('gen',1) , ('gn',1), ('bereshit', 1), ('exodus',2), ('exod',2), ('ex',2), ('shemot',2), ('leviticus',3), ('lev',3), ('lv',3), ('vayikra',3), ('numbers',4), ('num',4), ('nm',4), ('bemidbar',4), ('deuteronomy',5), ('deut',5), ('dt',5), ('devarim',5), ('joshua',6), ('josh',6), ('yehoshua',6), ('judges',7), ('judg',7), ('jgs',7), ('shoftim',7), ('ruth',8), ('ru',8), ('1 samuel',9), ('1samuel',9), ('1 sam',9), ('1sam',9), ('1 sm',9), ('1sm',9), ('1 shmuel',9), ('1shmuel',9), ('2 samuel',10), ('2samuel',10), ('2 sam',10), ('2sam',10), ('2 sm',10), ('2sm',10), ('2 shmuel',10), ('2shmuel',10), ('1 kings',11), ('1kings',11), ('1 kgs',11), ('1kgs',11), ('1 melachim',11), ('1melachim',11), ('2 kings',12), ('2kings',12), ('2 kgs',12), ('2kgs',12), ('2 melachim',12), ('2melachim',12), ('1 chronicles',13), ('1chronicles',13), ('1 chron',13), ('1chron',13), ('1 chr',13), ('1chr',13), ('2 chronicles',14), ('2chronicles',14), ('2 chron',14), ('2chron',14), ('2 chr',14), ('2chr',14), ('ezra',15), ('ezr',15), ('nehemiah',16), ('neh',16), ('esther',17), ('est',17), ('job',18), ('jb',18), ('psalms',19), ('psalm',19), ('ps',19), ('pss',19), ('proverbs',20), ('prov',20), ('prv',20), ('ecclesiastes',21), ('eccles',21), ('eccl',21), ('song of solomon',22), ('song of songs',22), ('song of sol',22), ('sg',22), ('isaiah',23), ('isa',23), ('yeshayahu',23), ('yeshaya',23), ('jeremiah',24), ('jer',24), ('yirmiyahu',24), ('yirmiyah',24), ('lamentations',25), ('lam',25), ('ezekiel',26), ('ezek',26), ('yechezkel',26), ('daniel',27), ('dan',27), ('dn',27), ('hosea',28), ('hos',28), ('hoshea',28), ('joel',29), ('jl',29), ('yoel',29), ('amos',30), ('am',30), ('obadiah',31), ('obad',31), ('ob',31), ('ovadiah',31), ('ovadyah',31), ('jonah',32), ('jon',32), ('micah',33), ('mic',33), ('michah',33), ('nahum',34), ('nah',34), ('na',34), ('nachum',34), ('habakkuk',35), ('hab',35), ('hb',35), ('chavakuk',35), ('zephaniah',36), ('zeph',36), ('zep',36), ('tzefaniah',36), ('tzefanyah',36), ('haggai',37), ('hag',37), ('hg',37), ('chaggai',37), ('zechariah',38), ('zech',38), ('zec',38), ('zecharya',38), ('zecharyah',38), ('zechariyah',38), ('malachi',39), ('mal',39), ('matthew',40), ('mathew',40), ('matt',40), ('mat',40), ('mt',40), ('mark',41), ('mk',41), ('luke',42), ('lk',42), ('john',43), ('jn',43), ('acts',44), ('acts of the apostles',44), ('romans',45), ('rom',45), ('1 corinthians',46), ('1corinthians',46), ('1 cor',46), ('1cor',46), ('2 corinthians',47), ('2corinthians',47), ('2 cor',47), ('2cor',47), ('galatians',48), ('gal',48), ('ephesians',49), ('philippians',50), ('phil',50), ('colossians',51), ('col',51), ('1 thessalonians',52), ('1thessalonians',52), ('1 thess',52), ('1thess',52), ('1 thes',52), ('1thes',52), ('2 thessalonians',53), ('2thessalonians',53), ('2 thess',53), ('2thess',53), ('2 thes',53), ('2thes',53), ('1 timothy',54), ('1timothy',54), ('1st timothy',54), ('1 tim',54), ('1tim',54), ('1 tm',54), ('1tm',54), ('2 timothy', 55), ('2timothy',55), ('2nd timothy',55), ('2 tim',55), ('2tim',55), ('2 tm',55), ('2tm',55), ('titus',56), ('ti',56), ('philemon',57), ('philem',57), ('phlm',57), ('hebrews',58), ('heb',58), ('james',59), ('jas',59), ('1 peter',60), ('1peter',60), ('1 pet',60), ('1pet',60), ('1 pt',60), ('1pt',60), ('2 peter',61), ('2peter',61), ('2 pet',61), ('2pet',61), ('2 pt',61), ('2pt',61), ('1 john',62), ('1john',62), ('1 jn',62), ('1jn',62), ('2 john',63), ('2john',63), ('2 jn',63), ('2jn',63), ('3 john',64), ('3john',64), ('3 jn',64), ('3jn',64), ('jude',65), ('revelation',66), ('revelations',66), ('rev',66), ('rv',66), ('judith',67), ('judeth',67), ('jdt',67), ('wisdom',68), ('wis',68), ('wisdom of solomon', 68), ('tobit',69), ('tob',69), ('sirach',70), ('sir',70), ('ecclesiasticus',70), ('baruch',71), ('bar',71), ('1 maccabees',72), ('1maccabees',72), ('1 macc',72), ('1macc',72), ('1 mac',72), ('1mac', 72), ('2 maccabees',73), ('2maccabees',73), ('2 macc', 73), ('2macc',73), ('2 mac',73), ('2mac',73), ('3 maccabees',74), ('3maccabees',74), ('3 macc',74), ('3macc',74), ('3 mac',74), ('3mac',74), ('4 maccabees',75), ('4maccabees',75), ('4 macc',75), ('4macc',75), ('4 mac',75), ('4mac',75), ('rest of daniel',76), ('additions to daniel',76), ('adddan',76), ('song of the three children',76), ('prayer of azariah',76), ('rest of esther',77), ('additions to esther',77), ('addesth',77), ('prayer of manasses',78), ('prayer of manasseh',78), ('manasses',78), ('manasseh',78), ('prman',78), ('1 esdras',79), ('1esdras',79), ('1 esd',79), ('1esd',79), ('2 esdras',80), ('2esdras',80), ('2 esd',80), ('2esd',80), ('story of susanna',81), ('susanna',81), ('sus',81), ('bel and the dragon',82), ('bel',82)]) sorted_books = OrderedDict(sorted(book_names.items(), key=lambda t: len(t[0]), reverse=True)) for key, value in sorted_books.items(): if search(r'\b' + key + r'\b', phrase): return value return False
def test_iterators(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) self.assertEqual(list(od), [t[0] for t in pairs]) self.assertEqual(list(od.keys()), [t[0] for t in pairs]) self.assertEqual(list(od.values()), [t[1] for t in pairs]) self.assertEqual(list(od.items()), pairs) self.assertEqual(list(reversed(od)), [t[0] for t in reversed(pairs)])
def test_reinsert(self): # Given insert a, insert b, delete a, re-insert a, # verify that a is now later than b. od = OrderedDict() od['a'] = 1 od['b'] = 2 del od['a'] od['a'] = 1 self.assertEqual(list(od.items()), [('b', 2), ('a', 1)])
def getFreeTextData(self, tests): data = OrderedDict() for testName, state, extraVersion in tests: freeText = state.freeText if hasattr(state, "freeText") else None if freeText: if not data.has_key(freeText): data[freeText] = [] data[freeText].append((testName, state, extraVersion)) return data.items()
def getFreeTextData(self): data = OrderedDict() for test in self.getAllTests(): freeText = test.state.freeText if freeText: if not data.has_key(freeText): data[freeText] = [] data[freeText].append(test) return data.items()
def _get_tab(self, first_id, last_id): """Return a list of selected items in a tab section of settings.""" selected_items = {} for id in range(first_id, last_id + 1): if self._xbmcaddon.getSetting(str(id)) == "true": selected_items.update({id: KEYWORDS[str(id)]}) # Sort dictionary by key. selected_items = OrderedDict(sorted(selected_items.items())) return selected_items
def getFreeTextData(self, tests): data = OrderedDict() for testName, state, extraVersion in tests: freeText = state.freeText if freeText: if not data.has_key(freeText): data[freeText] = [] data[freeText].append((testName, state, extraVersion)) return data.items()
def __add_parser(self, config_path, parent=None): # Get configuration elements from config_path. config = self.__get_config(config_path) self.__check_parser_conf(config_path, config) # Init main parser. if parent is None: self.parser = argparse.ArgumentParser(add_help=True) parent = self.parser # We may need to access to the parser object later (for printing usage # by example) so save it to __parsers attibute. self.__parsers.setdefault('/'.join(config_path), parent) # Add subparser. if config.get('subparsers'): subparsers_config = config.get('subparsers', {}) subparser_dest = '%s%d' % (self.keyword, len(config_path) / 2) subparsers = parent.add_subparsers(dest=subparser_dest) subparsers.required = True for subparser_name in subparsers_config: subparser_path = list(config_path) subparser_path.extend(['subparsers', subparser_name]) subparser = subparsers.add_parser(subparser_name, add_help=True) subparser.name = subparser_name self.__add_parser(subparser_path, subparser) else: # For options, use a deep copy for not altering initial # configuration when change must be done. options_config = copy.deepcopy(config.get('options', {})) args_config = OrderedDict(config.get('args', {})) if 'usage' in config: parent.usage = self.__usage(parent.prog, config['usage']) if 'desc' in config: parent.description = config['desc'] try: # Add groups. for group_type in 'groups', 'exclusive_groups': if config.get(group_type, {}): self.__add_groups( parent, config.get(group_type), options_config, True if group_type == 'exclusive_groups' else False ) # Add options. for option, option_config in options_config.items(): self.__add_option(parent, option, option_config) # Add args: for arg, arg_config in args_config.items(): self.__add_option(parent, arg, arg_config, True) except CLGError as err: raise CLGError('/%s: %s' % ('/'.join(config_path), err))
def dict2rec(*args, **kwargs): """ Convert dict of arrays to nested recarray. The input can be anything accepted by :meth:`OrderedDict`, see :class:`dict`. All items must be the same length, which becomes the length of the resulting recarray. >>> dict2rec(zip(("a", "b", "c"), ... (np.arange(0, 3), np.arange(10, 13), np.zeros(3)))) array([(0, 10, 0.0), (1, 11, 0.0), (2, 12, 0.0)], dtype=[('a', '<i...'), ('b', '<i...'), ('c', '<f8')]) >>> dict2rec(OrderedDict(zip(("a", "b", "c"), ... (np.arange(0, 3), np.arange(10, 13), np.zeros(3))))) array([(0, 10, 0.0), (1, 11, 0.0), (2, 12, 0.0)], dtype=[('a', '<i...'), ('b', '<i...'), ('c', '<f8')]) >>> dict2rec({ ... "a": np.arange(0, 3), "b": np.arange(10, 13), "c": np.zeros((3,))}) array([(0, 0.0, 10), (1, 0.0, 11), (2, 0.0, 12)], dtype=[('a', '<i...'), ('c', '<f8'), ('b', '<i...')]) Vector-valued fields. >>> dict2rec([("a", [0, 1, 2]), ... ("b", [10, 11, 12]), ... ("c", [(0, 1), (2, 3), (4, 5)]), ... ]) array([(0, 10, [0, 1]), (1, 11, [2, 3]), (2, 12, [4, 5])], dtype=[('a', '<i4'), ('b', '<i4'), ('c', '<i4', (2,))]) """ d = OrderedDict(*args, **kwargs) for k, v in d.items(): d[k] = np.atleast_1d(v) # Convert keys to plain str because Numpy field names cannot be unicode dtype = [(str(k), v.dtype, v.shape[1:]) for k, v in d.items()] shape = len(v) x = np.zeros(shape=shape, dtype=dtype) for k, v in d.items(): x[k] = v return x
def list_ (request): priority = map(int, request.GET.getall('pr')) tags = request.GET.get('tags', '') def repl (matchobj): m = matchobj.group(0) if m in ('or', 'and', 'not', '(', ')'): return m.upper() else: return u"tags LIKE '%%#%s#%%'" % m tags = re.sub(RE_TAG_Q, repl, tags) entries = OrderedDict() q = Entry.query.order_by(Entry.priority.desc(), Entry.created_time.desc()) if tags: q = q.filter(tags) if priority: q = q.filter(Entry.priority.in_(priority)) for e in q: e.children = [] entries[e.id] = e keys_to_del = [] items = entries.items() for id, e in items: if e.parent_id: keys_to_del.append(id) if e.parent_id not in entries: #for filter entries[e.parent_id] = e.parent e.parent.children = [] items.append((e.parent_id, e.parent)) #visit parents too later so hierarchy doesnt breake entries[e.parent_id].children.append(e) for id in keys_to_del: del entries[id] if tags: entries = OrderedDict(sorted(entries.items(), key = (lambda (k, e): e.priority), reverse = True)) return { 'entries': entries.values(), 'priorities': priorities, }
def generate_topic_word(self): 'generate estimated topics for a document' from ordereddict import OrderedDict # build topic-vocab map topic_vocab = {} for word_vocab_id in self.vocab_topic: vt = self.vocab_topic[word_vocab_id] sorted_vt = OrderedDict(sorted(vt.items(), key=lambda x: -x[1])) for svt, (k, v) in enumerate(sorted_vt.items()): if svt > 10: break else: if k not in topic_vocab: vocab_list = {} vocab_list[word_vocab_id] = v else: vocab_list = topic_vocab[k] if word_vocab_id in vocab_list: old_association = vocab_list[word_vocab_id] if old_association < v: vocab_list[word_vocab_id] = v else: vocab_list[word_vocab_id] = v topic_vocab[k] = vocab_list # combine topic-vocab with index_vocab to create a topic_id --> vocabulary list map topic_word = {} for topic_id in topic_vocab: topic_word[topic_id] = [] tv = topic_vocab[topic_id] sorted_tv = OrderedDict(sorted(tv.items(), key=lambda x: -x[1])) for stv, (k, v) in enumerate(sorted_tv.items()): if stv > 10: break else: topic_word[topic_id].append(self.index_vocab[k]) return topic_word '''# publish
def writeErrors(self, rejectionInfo): # Don't write errors if only some of a group are rejected appsByName = OrderedDict() rejectedApps = set(rejectionInfo.keys()) for suite in self.suites: app = suite.app appsByName.setdefault(app.name, []).append(app) for _, appGroup in appsByName.items(): if set(appGroup).issubset(rejectedApps): for app in appGroup: if app in rejectionInfo: sys.stderr.write(app.rejectionMessage(rejectionInfo.get(app)))
def findAllClasses(self, className): if len(self.allApps) == 0: return [ (className, []) ] else: classNames = OrderedDict() for app in self.allApps: allConfigsForApp = self.getAllIntvConfigs([ app ]) replacements = plugins.ResponseAggregator([ x.getReplacements for x in allConfigsForApp])() for config in allConfigsForApp: if className in config.getInteractiveActionClasses(self.dynamic): realClassName = replacements.get(className, className) classNames.setdefault(realClassName, []).append(app) return classNames.items()
def test_upload(req): var_info = OrderedDict(( ('form_keys', req.form.keys()), ('form_value_names', ', '.join(tuple(req.form[key].name for key in req.form.keys()))), ('form_value_types', ', '.join(tuple(req.form[key].type for key in req.form.keys()))), ('form_value_fnames', ', '.join(tuple(uni(req.form[key].filename) for key in req.form.keys()))), ('form_value_lenghts', ', '.join(tuple(uni(req.form[key].length) for key in req.form.keys()))), ('form_value_files', ', '.join(tuple(uni(req.form[key].file) for key in req.form.keys()))), ('form_value_lists', ', '.join(tuple( 'Yes' if req.form[key].list else 'No' for key in req.form.keys()))), )) files = [] for key in req.form.keys(): if req.form[key].filename: files.append("<h2>%s</h2>" % req.form[key].filename) files.append("<i>%s</i>" % req.form[key].type) if req.form[key].type.startswith('text/'): files.append("<pre>%s</pre>" % html(req.form.getvalue(key).decode('utf-8'))) else: files.append("<pre>%s</pre>" % encodestring(req.form.getvalue(key)).decode()) buff = get_header('HTTP file upload test') + \ (get_crumbnav(req), "<h2>Upload Form</h2>", '<form method="post" enctype="multipart/form-data">', '<input type="file" name="file_0"><br/>', '<input type="file" name="file_1"><br/>', '<input type="file" name="file_2"><br/>', '<input type="submit" name="btn" value="Upload">' '</form>', "<h2>Uploaded File</h2>", "<table>") + \ tuple("<tr><td>%s:</td><td>%s</td></tr>" % (key, html(val)) for key, val in var_info.items()) + \ ("</table>",) + \ tuple(files) + \ get_footer() for line in buff: req.write(line + '\n') return state.OK
def get_folder_items(self): folders = OrderedDict() items = OrderedDict() for item in self.context.listFolderContents(): item_id = item.getId() item_dict = {"id" : item_id, "title": item.Title(), "description": item.Description(), "item_url": item.absolute_url(), "image_url": "" , "portal_type": item.portal_type} if item_dict["portal_type"] in ["Folder",]: folders[item_id] = item_dict for folder_item in item.listFolderContents(): if folder_item.portal_type == "Image": folders[item_id]["image_url"] = \ folder_item.absolute_url() break elif item_dict["portal_type"] in ["Image"]: item_dict["image_url"] = item.absolute_url() items[item_id] = item_dict else: items[item_id] = item_dict return OrderedDict(folders.items() + items.items())
def findAllClasses(self, className): if len(self.allApps) == 0: return [(className, [])] else: classNames = OrderedDict() for app in self.allApps: allConfigsForApp = self.getAllIntvConfigs([app]) replacements = plugins.ResponseAggregator( [x.getReplacements for x in allConfigsForApp])() for config in allConfigsForApp: if className in config.getInteractiveActionClasses( self.dynamic): realClassName = replacements.get(className, className) classNames.setdefault(realClassName, []).append(app) return classNames.items()
class DictCache(AbstractCache): """Implementation of a cache in a memory dictionary """ def __init__(self, ttl=60): """Creates a new instance params: ``ttl`` Time to live of the data. """ super(DictCache, self).__init__(ttl=ttl) try: self._ich = collections.OrderedDict() self._ttds = collections.OrderedDict() except AttributeError: #This version of python does not support OrderedDict from ordereddict import OrderedDict self._ich = OrderedDict() self._ttds = OrderedDict() def store_data(self, k, ttl, v): self._ich[k] = v self._ttds[k] = ( time.time() + ttl if ttl != None else None ) def retrieve_data(self, k): ttd = self._ttds.get(k, 0) if ttd == None or time.time() < ttd: return self._ich[k] elif ttd: self._ttds.pop(k) self._ich.pop(k) def clear_expired(self): for k, ttd in self._ttds.items(): if ttd != None and ttd < time.time(): self._ttds.pop(k) self._ich.pop(k) else: break def clear(self): self._ich.clear() self._ttds.clear()
def add_folders(self, folders, sorted_by_values=True): """Add many folders to a menu. Can be sorted by folder name.""" translations = {} for id in folders: value = self.xbmcaddon.translate(id) translations.update({id: value}) if sorted_by_values: # Default. # IMPORTANT! This is the suggested way by Python 2.7, but it seems # to have problems when first letter is accented. translations = \ OrderedDict(sorted(translations.items(), key=lambda t: t[1])) for id, folder in translations.iteritems(): self._add_folder(id, folder, folders[id]) else: # Sorted by key. for id, folder in folders.iteritems(): self._add_folder(id, translations[id], folder)
def begetPathArgs(self, data): """ Create pargs for url path generation from pkeys if given Do urlencoding return pargs dict """ pargs = ODict() # assign values from kwa for pkeys for key in self.pkeys: try: pargs[key] = data[key] except KeyError as ex: raise KeyEmptorError(msg = "Missing data for path key.",key=key) for key, val in pargs.items(): pargs[key] = urllib.quote(val, '/:;#') return pargs
def executeCommand(envvars): buildenv = os.environ env = OrderedDict() for name, value in envvars: if name == 'COMMAND': #commands = value.split(';') commands = value; else: env[name] = value handleDollarValues(env) for key, value in env.items(): buildenv[key] = value p = subprocess.Popen("%s" % (commands), shell=True, executable = "/bin/sh", env=buildenv) sts = os.waitpid(p.pid, 0)[1] if ((sts/256) != 0): return 0 return 1
def expandCollations(self, test): newColl = OrderedDict() coll = test.getConfigValue("collate_file") self.diag.info("coll initial:" + str(coll)) for targetPattern in sorted(coll.keys()): sourcePatterns = coll.get(targetPattern) if not glob.has_magic(targetPattern): newColl[targetPattern] = sourcePatterns continue # add each file to newColl by transferring wildcards across for sourcePattern in sourcePatterns: for sourcePath in self.findPaths(test, sourcePattern): # Use relative paths: easier to debug and SequenceMatcher breaks down if strings are longer than 200 chars relativeSourcePath = plugins.relpath(sourcePath, test.getDirectory(temporary=1)) newTargetStem = self.makeTargetStem(targetPattern, sourcePattern, relativeSourcePath) self.diag.info("New collation to " + newTargetStem + " : from " + relativeSourcePath) newColl.setdefault(newTargetStem, []).append(sourcePath) return newColl.items()
def get(self, row, columns=None, column_start=None, super_column=None, column_finish=None, column_count=100): try: if super_column is None: data_columns = self.data[row] else: data_columns = self.data[row][super_column] results = OrderedDict() count = 0 if columns is not None: for c in columns: results[c] = data_columns[c] else: for c in sorted(data_columns.keys()): if count > column_count: break if column_start and cmp(c, column_start) < 0: continue if column_finish and cmp(c, column_finish) > 0: break results[c] = data_columns[c] count += 1 if not len(results): raise NotFoundException for key, value in results.items(): if isinstance(value, dict) and len(value) == 0: del (results[key]) if value is None: del (results[key]) return results except KeyError: raise NotFoundException
def test_copying(self): # Check that ordered dicts are copyable, deepcopyable, picklable, # and have a repr/eval round-trip pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] od = OrderedDict(pairs) update_test = OrderedDict() update_test.update(od) for i, dup in enumerate([ od.copy(), copy.copy(od), copy.deepcopy(od), pickle.loads(pickle.dumps(od, 0)), pickle.loads(pickle.dumps(od, 1)), pickle.loads(pickle.dumps(od, 2)), pickle.loads(pickle.dumps(od, -1)), eval(repr(od)), update_test, OrderedDict(od), ]): self.assert_(dup is not od) self.assertEquals(dup, od) self.assertEquals(list(dup.items()), list(od.items())) self.assertEquals(len(dup), len(od)) self.assertEquals(type(dup), type(od))
class CategoryHandler: def __init__(self): self.testsInCategory = OrderedDict() def update(self, categoryHandler): for category, testInfo in categoryHandler.testsInCategory.items(): testInfoList = self.testsInCategory.setdefault(category, []) testInfoList += testInfo def registerInCategory(self, testId, category, extraVersion, state): self.testsInCategory.setdefault(category, []).append((testId, state, extraVersion)) def getDescription(self, cat, count): shortDescr, _ = getCategoryDescription(cat) return str(count) + " " + shortDescr def getTestCountDescription(self, count): return str(count) + " tests: " def generateTextSummary(self): numTests, summaryData = self.getSummaryData() categoryDescs = [ self.getDescription(cat, count) for cat, count in summaryData ] return self.getTestCountDescription(numTests) + " ".join(categoryDescs) def generateHTMLSummary(self, detailPageRef, extraVersion=None): numTests, summaryData = self.getSummaryData(extraVersion) container = HTMLgen.Container() for cat, count in summaryData: summary = HTMLgen.Text(self.getDescription(cat, count)) if cat == "success": container.append(summary) else: linkTarget = detailPageRef + getCategoryDescription(cat)[-1] container.append(HTMLgen.Href(linkTarget, summary)) testCountSummary = HTMLgen.Text(self.getTestCountDescription(numTests)) return HTMLgen.Container(testCountSummary, container) def countTests(self, testInfo, extraVersion): if extraVersion is not None: return sum((currExtra == extraVersion for (testId, state, currExtra) in testInfo)) else: return len(testInfo) def getSummaryData(self, extraVersion=None): numTests = 0 summaryData = [] for cat, testInfo in self.testsInCategory.items(): testCount = self.countTests(testInfo, extraVersion) if testCount > 0: summaryData.append((cat, testCount)) numTests += testCount summaryData.sort(key=self.getSummarySortKey) return numTests, summaryData def getTestsWithDescriptions(self): return sorted([ (getCategoryDescription(cat)[1], testInfo) for cat, testInfo in self.testsInCategory.items() ]) def getSummarySortKey(self, data): # Put success at the start, it's neater like that return data[0] != "success", -data[1], data[0]
class PluginManager(object): ENVIRONMENT_HOOK = 'haas.hooks.environment' RESULT_HANDLERS = 'haas.result.handler' TEST_RUNNER = 'haas.runner' TEST_DISCOVERY = 'haas.discovery' _help = { TEST_DISCOVERY: 'The test discovery implementation to use.', TEST_RUNNER: 'Test runner implementation to use.', RESULT_HANDLERS: 'Test result handler implementation to use.', } _namespace_to_option_parts = { TEST_DISCOVERY: ['discovery'], TEST_RUNNER: ['runner'], RESULT_HANDLERS: ['result', 'handler'], } def __init__(self): self.hook_managers = OrderedDict() self.hook_managers[self.ENVIRONMENT_HOOK] = ExtensionManager( namespace=self.ENVIRONMENT_HOOK, ) self.hook_managers[self.RESULT_HANDLERS] = ExtensionManager( namespace=self.RESULT_HANDLERS, ) self.driver_managers = OrderedDict() self.driver_managers[self.TEST_DISCOVERY] = ExtensionManager( namespace=self.TEST_DISCOVERY, ) self.driver_managers[self.TEST_RUNNER] = ExtensionManager( namespace=self.TEST_RUNNER, ) @classmethod def testing_plugin_manager(cls, hook_managers, driver_managers): """Create a fabricated plugin manager for testing. """ plugin_manager = cls.__new__(cls) plugin_manager.hook_managers = OrderedDict(hook_managers) plugin_manager.driver_managers = OrderedDict(driver_managers) return plugin_manager def _hook_extension_option_prefix(self, extension): name = uncamelcase(extension.name, sep='-').replace('_', '-') option_prefix = '--with-{0}'.format(name) dest_prefix = name.replace('-', '_') return option_prefix, dest_prefix def _namespace_to_option(self, namespace): parts = self._namespace_to_option_parts[namespace] option = '--{0}'.format('-'.join(parts)) dest = '_'.join(parts) return option, dest def _add_hook_extension_arguments(self, extension, parser): option_prefix, dest_prefix = self._hook_extension_option_prefix( extension) extension.plugin.add_parser_arguments( parser, extension.name, option_prefix, dest_prefix) def _create_hook_plugin(self, extension, args, **kwargs): option_prefix, dest_prefix = self._hook_extension_option_prefix( extension) plugin = extension.plugin.from_args( args, extension.name, dest_prefix, **kwargs) if plugin is not None and plugin.enabled: return plugin return None def _add_driver_extension_arguments(self, extension, parser, option_prefix, dest_prefix): extension.plugin.add_parser_arguments( parser, option_prefix, dest_prefix) def add_plugin_arguments(self, parser): """Add plugin arguments to argument parser. Parameters ---------- parser : argparse.ArgumentParser The main haas ArgumentParser. """ for manager in self.hook_managers.values(): if len(list(manager)) == 0: continue manager.map(self._add_hook_extension_arguments, parser) for namespace, manager in self.driver_managers.items(): choices = list(sorted(manager.names())) if len(choices) == 0: continue option, dest = self._namespace_to_option(namespace) parser.add_argument( option, help=self._help[namespace], dest=dest, choices=choices, default='default') option_prefix = '{0}-'.format(option) dest_prefix = '{0}_'.format(dest) manager.map(self._add_driver_extension_arguments, parser, option_prefix, dest_prefix) def get_enabled_hook_plugins(self, hook, args, **kwargs): """Get enabled plugins for specified hook name. """ manager = self.hook_managers[hook] if len(list(manager)) == 0: return [] return [ plugin for plugin in manager.map( self._create_hook_plugin, args, **kwargs) if plugin is not None ] def get_driver(self, namespace, parsed_args, **kwargs): """Get mutually-exlusive plugin for plugin namespace. """ option, dest = self._namespace_to_option(namespace) dest_prefix = '{0}_'.format(dest) driver_name = getattr(parsed_args, dest, 'default') driver_extension = self.driver_managers[namespace][driver_name] return driver_extension.plugin.from_args( parsed_args, dest_prefix, **kwargs)
did_you_mean.append(query_word+query_separators[i]) did_you_mean_markdown.append(query_word+query_separators[i]) did_you_mean_html.append(query_word+query_separators[i]) did_you_mean = "".join(did_you_mean) did_you_mean_markdown = "".join(did_you_mean_markdown) did_you_mean_html = "".join(did_you_mean_html) output = OrderedDict() output["your_input"] = query output["did_you_mean"] = did_you_mean output["did_you_mean_markdown"] = did_you_mean_markdown output["did_you_mean_html"] = did_you_mean_html self.stdout.write('') self.stdout.write("Result:") for i,k in output.items(): self.stdout.write(" '%s': %s" % (i,k.encode("utf-8",errors="replace"),)) self.stdout.write('') return output pass def do_import_file(self,filepath,locale): with codecs.open(filepath, encoding='utf-8') as file: for line in file: sentences, separators_s = self.split_bysentences(line) if len([sentence for sentence in sentences if sentence]): for sentence in sentences: self.import_sentence(sentence,locale) pass
class HasParameters(object): """This class provides an implementation of the IHasParameters interface.""" _do_not_promote = [ 'get_expr_depends', 'get_referenced_compnames', 'get_referenced_varpaths', 'get_metadata' ] def __init__(self, parent): self._parameters = OrderedDict() self._allowed_types = ['continuous'] if obj_has_interface(parent, ISolver): self._allowed_types.append('unbounded') self._parent = None if parent is None else weakref.ref(parent) def __getstate__(self): state = self.__dict__.copy() state['_parent'] = self.parent return state def __setstate__(self, state): self.__dict__.update(state) parent = state['_parent'] self._parent = None if parent is None else weakref.ref(parent) @property def parent(self): """ The object we are a delegate of. """ return None if self._parent is None else self._parent() def _item_count(self): """This is used by the replace function to determine if a delegate from the target object is 'empty' or not. If it's empty, it's not an error if the replacing object doesn't have this delegate. """ return len(self._parameters) def add_parameter(self, target, low=None, high=None, scaler=None, adder=None, start=None, fd_step=None, name=None, scope=None): """Adds a parameter or group of parameters to the driver. target: string or iter of strings or Parameter What the driver should vary during execution. A *target* is an expression that can reside on the left-hand side of an assignment statement, so typically it will be the name of a variable or possibly a subscript expression indicating an entry within an array variable, e.g., x[3]. If an iterator of targets is given, then the driver will set all targets given to the same value whenever it varies this parameter during execution. If a Parameter instance is given, then that instance is copied into the driver with any other arguments specified, overiding the values in the given parameter. low: float (optional) Minimum allowed value of the parameter. If scaler and/or adder is supplied, use the transformed value here. If target is an array, this may also be an array, but must have the same size. high: float (optional) Maximum allowed value of the parameter. If scaler and/or adder is supplied, use the transformed value here. If target is an array, this may also be an array, but must have the same size. scaler: float (optional) Value to multiply the possibly offset parameter value by. If target is an array, this may also be an array, but must have the same size. adder: float (optional) Value to add to parameter prior to possible scaling. If target is an array, this may also be an array, but must have the same size. start: any (optional) Value to set into the target or targets of a parameter before starting any executions. If not given, analysis will start with whatever values are in the target or targets at that time. If target is an array, this may also be an array, but must have the same size. fd_step: float (optional) Step-size to use for finite difference calculation. If no value is given, the differentiator will use its own default. If target is an array, this may also be an array, but must have the same size. name: str (optional) Name used to refer to the parameter in place of the name of the variable referred to in the parameter string. This is sometimes useful if, for example, multiple entries in the same array variable are declared as parameters. scope: object (optional) The object to be used as the scope when evaluating the expression. If neither "low" nor "high" is specified, the min and max will default to the values in the metadata of the variable being referenced. """ if isinstance(target, (ParameterBase, ParameterGroup)): self._parameters[target.name] = target target.override(low, high, scaler, adder, start, fd_step, name) else: if isinstance(target, basestring): names = [target] key = target else: names = target key = tuple(target) if name is not None: key = name dups = set(self.list_param_targets()).intersection(names) if len(dups) == 1: self.parent.raise_exception( "'%s' is already a Parameter" " target" % dups.pop(), ValueError) elif len(dups) > 1: self.parent.raise_exception( "%s are already Parameter targets" % sorted(list(dups)), ValueError) if key in self._parameters: self.parent.raise_exception("%s is already a Parameter" % key, ValueError) try: _scope = self._get_scope(scope) if len(names) == 1: target = self._create(names[0], low, high, scaler, adder, start, fd_step, key, _scope) else: # defining a ParameterGroup parameters = [ self._create(n, low, high, scaler, adder, start, fd_step, key, _scope) for n in names ] types = set([p.valtypename for p in parameters]) if len(types) > 1: raise ValueError("Can't add parameter %s because " "%s are not all of the same type" % (key, " and ".join(names))) target = ParameterGroup(parameters) self._parameters[key] = target except Exception: self.parent.reraise_exception(info=sys.exc_info()) self.parent.config_changed() def _create(self, target, low, high, scaler, adder, start, fd_step, key, scope): """ Create one Parameter or ArrayParameter. """ try: expreval = ExprEvaluator(target, scope) except Exception as err: raise err.__class__("Can't add parameter: %s" % err) if not expreval.is_valid_assignee(): raise ValueError("Can't add parameter: '%s' is not a" " valid parameter expression" % expreval.text) try: val = expreval.evaluate() except Exception as err: val = None # Let Parameter code sort out why. name = key[0] if isinstance(key, tuple) else key if isinstance(val, ndarray): return ArrayParameter(target, low=low, high=high, scaler=scaler, adder=adder, start=start, fd_step=fd_step, name=name, scope=scope, _expreval=expreval, _val=val, _allowed_types=self._allowed_types) else: return Parameter(target, low=low, high=high, scaler=scaler, adder=adder, start=start, fd_step=fd_step, name=name, scope=scope, _expreval=expreval, _val=val, _allowed_types=self._allowed_types) def remove_parameter(self, name): """Removes the parameter with the given name.""" param = self._parameters.get(name) if param: del self._parameters[name] else: self.parent.raise_exception( "Trying to remove parameter '%s' " "that is not in this driver." % (name, ), AttributeError) self.parent.config_changed() def config_parameters(self): """Reconfigure parameters from potentially changed targets.""" for param in self._parameters.values(): param.configure() def get_references(self, name): """Return references to component `name` in preparation for subsequent :meth:`restore_references` call. name: string Name of component being removed. """ refs = OrderedDict() for pname, param in self._parameters.items(): if name in param.get_referenced_compnames(): refs[pname] = param return refs def remove_references(self, name): """Remove references to component `name`. name: string Name of component being removed. """ to_remove = [] for pname, param in self._parameters.items(): if name in param.get_referenced_compnames(): to_remove.append(pname) for pname in to_remove: self.remove_parameter(pname) def restore_references(self, refs): """Restore references to component `name` from `refs`. refs: object Value returned by :meth:`get_references`. """ for pname, param in refs.items(): try: self.add_parameter(param) except Exception as err: self.parent._logger.warning( "Couldn't restore parameter '%s': %s" % (pname, str(err))) def list_param_targets(self): """Returns a list of parameter targets. Note that this list may contain more entries than the list of Parameter, ParameterGroup, and ArrayParameter objects since ParameterGroup instances have multiple targets. """ targets = [] for param in self._parameters.values(): targets.extend(param.targets) return targets def list_param_group_targets(self): """Returns a list of tuples that contain the targets for each parameter group. """ targets = [] for param in self.get_parameters().values(): targets.append(tuple(param.targets)) return targets def clear_parameters(self): """Removes all parameters.""" for name in self._parameters.keys(): self.remove_parameter(name) self._parameters = OrderedDict() def get_parameters(self): """Returns an ordered dict of parameter objects.""" return self._parameters def total_parameters(self): """Returns the total number of values to be set.""" return sum([param.size for param in self._parameters.values()]) def init_parameters(self): """Sets all parameters to their start value if a start value is given """ scope = self._get_scope() for param in self._parameters.itervalues(): if param.start is not None: param.set(param.start, scope) def set_parameter_by_name(self, name, value, case=None, scope=None):
id = row[0] body = row[1] title = row[2] tag_string = row[3] processed_body = remove_tags(remove_code(body)) processed_tags = TAG_EXTRACTOR.findall(tag_string) features = OrderedDict() features["textFeatures"] = calculate_text_features(body, title) features["tagFeatures"] = calculate_tag_features(processed_tags) features["shallowLinguisticFeatures"] = TextStatistics( processed_body).calculate_shallow_text_features() # convert ordereddicts to list values = [] for category, feature in features.items(): for value in feature.values(): if isinstance(value, list): values.extend(value) # arrays else: values.append(value) # single Integers, Floats values.append(id) updateData.append(values) if len(updateData) > FLUSH_LIMIT: update_trainings_features(updateData, cursor, cnx) updateData = []
class Config(object): """ KGEN configuration parameter holder """ def __init__(self): # setup config parameters self._attrs = OrderedDict() self.opt_handlers = OrderedDict() # KGEN operation mode self._attrs['check_mode'] = False # Fortran parameters self._attrs['fort'] = OrderedDict() self._attrs['fort']['maxlinelen'] = 132 # logging parameters self._attrs['logging'] = OrderedDict() self._attrs['logging']['select'] = OrderedDict() # callsite parameters self._attrs['callsite'] = OrderedDict() self._attrs['callsite']['filepath'] = '' self._attrs['callsite']['span'] = (-1, -1) self._attrs['callsite']['namepath'] = '' # self._attrs['callsite']['lineafter'] = -1 # external tool parameters self._attrs['bin'] = OrderedDict() self._attrs['bin']['pp'] = 'cpp' self._attrs['bin']['cpp_flags'] = '-w -traditional' self._attrs['bin']['fpp_flags'] = '-w' # search parameters self._attrs['search'] = OrderedDict() self._attrs['search']['skip_intrinsic'] = True self._attrs['search']['except'] = [] self._attrs['search']['promote_exception'] = False # path parameters self._attrs['path'] = OrderedDict() self._attrs['path']['outdir'] = '.' self._attrs['path']['state'] = 'state' self._attrs['path']['kernel'] = 'kernel' # source file parameters self._attrs['source'] = OrderedDict() self._attrs['source']['isfree'] = None self._attrs['source']['isstrict'] = None self._attrs['source']['alias'] = OrderedDict() self._attrs['source']['file'] = OrderedDict() # include parameters self._attrs['include'] = OrderedDict() self._attrs['include']['macro'] = OrderedDict() self._attrs['include']['path'] = ['.'] self._attrs['include']['type'] = OrderedDict() self._attrs['include']['import'] = OrderedDict() self._attrs['include']['file'] = OrderedDict() # add mpi frame code in kernel driver self._attrs['add_mpi_frame'] = OrderedDict() self._attrs['add_mpi_frame']['enabled'] = False self._attrs['add_mpi_frame']['np'] = '2' self._attrs['add_mpi_frame']['mpiexec'] = 'mpiexec' # exclude parameters self._attrs['exclude'] = OrderedDict() # debugging parameters self._attrs['debug'] = OrderedDict() self._attrs['debug']['printvar'] = [] # plugin parameters self._attrs['plugin'] = OrderedDict() self._attrs['plugin']['priority'] = OrderedDict() def apply(self, cfg): import optparse if not cfg: raise ProgramException('Custom configuration is not provided.') if hasattr(cfg, 'attrs'): self._attrs.update(cfg.attrs) # parsing arguments parser = optparse.OptionParser(usage=cfg.usage, version=cfg.version) # add default options parser.add_option("-s", "--syntax-check", dest="syntax_check", action='store_true', default=False, help="KGEN Syntax Check Mode") parser.add_option("-i", "--include-ini", dest="include_ini", action='store', type='string', default=None, help="information used for analysis") parser.add_option("-e", "--exclude-ini", dest="exclude_ini", action='store', type='string', default=None, help="information excluded for analysis") parser.add_option("-I", dest="include", action='append', type='string', default=None, help="include path information used for analysis") parser.add_option("-D", dest="macro", action='append', type='string', default=None, help="macro information used for analysis") parser.add_option("--outdir", dest="outdir", action='store', type='string', default=None, help="path to create outputs") parser.add_option("--source", dest="source", action='append', type='string', default=None, help="Setting source file related properties") parser.add_option("--skip-intrinsic", dest="skip_intrinsic", action='store_true', default=False, help=optparse.SUPPRESS_HELP) parser.add_option("--noskip-intrinsic", dest="noskip_intrinsic", action='store_true', default=False, help=optparse.SUPPRESS_HELP) parser.add_option( "--intrinsic", dest="intrinsic", action='append', type='string', default=None, help= "Specifying resolution for intrinsic procedures during searching") parser.add_option("--debug", dest="debug", action='append', type='string', help=optparse.SUPPRESS_HELP) parser.add_option("--logging", dest="logging", action='append', type='string', help=optparse.SUPPRESS_HELP) parser.add_option("--add-mpi-frame", dest="add_mpi_frame", type='string', default=None, help='Add MPI frame codes in kernel_driver.') # add custom options if hasattr(cfg, 'options'): for opt_handler, args, kwargs in cfg.options: parser.add_option(*args, **kwargs) self.opt_handlers[kwargs['dest']] = opt_handler opts, args = parser.parse_args() if len(args) < 1: print 'ERROR: No call-site information is provided in command line.' sys.exit(-1) if opts.syntax_check: self._process_default_flags(opts) self._attrs['check_mode'] = args return # old options if opts.skip_intrinsic: print "skip-intrinsic flag is discarded. Please use --intrinsic skip instead" sys.exit(-1) if opts.noskip_intrinsic: print "noskip-intrinsic flag is discarded. Please use --intrinsic noskip instead" sys.exit(-1) callsite = args[0].split(':', 1) if not os.path.isfile(callsite[0]): print 'ERROR: %s can not be found.' % callsite[0] sys.exit(-1) # set callsite filepath self.callsite['filepath'] = callsite[0] # set namepath if exists in command line argument if len(callsite) == 2: self.callsite['namepath'] = callsite[1].lower() elif len(callsite) > 2: print 'ERROR: Unrecognized call-site information(Syntax -> filepath[:subprogramname]): %s' % str( callsite) sys.exit(-1) # process default flags self._process_default_flags(opts) # process custom flags for optname, opt_handler in self.opt_handlers.items(): opt = getattr(opts, optname, None) if opt and optname in self.opt_handlers: self.opt_handlers[optname](opt) def _process_default_flags(self, opts): # check if exists fpp or cpp output = '' try: output = exec_cmd('which cpp', show_error_msg=False).strip() except Exception as e: pass if output.endswith('cpp'): self.bin['pp'] = output else: output = '' try: output = exec_cmd('which fpp', show_error_msg=False).strip() except Exception as e: pass if output.endswith('fpp'): self.bin['pp'] = output else: print 'ERROR: neither cpp or fpp is found' sys.exit(-1) # parsing intrinsic skip option if opts.intrinsic: subflags = [] for line in opts.intrinsic: subflags.extend(line.split(',')) for subf in subflags: if subf and subf.find('=') > 0: key, value = subf.split('=') if key == 'except': self._attrs['search']['except'].extend( value.split(';')) elif key == 'add_intrinsic': Intrinsic_Procedures.extend( [name.lower() for name in value.split(';')]) else: raise UserException( 'Unknown intrinsic sub option: %s' % subf) else: if subf == 'skip': self._attrs['search']['skip_intrinsic'] = True elif subf == 'noskip': self._attrs['search']['skip_intrinsic'] = False else: raise UserException( 'Unknown intrinsic option(s) in %s' % subf) # parsing include parameters if opts.include: for inc in opts.include: inc_eq = inc.split('=') if len(inc_eq) == 1: for inc_colon in inc_eq[0].split(':'): self._attrs['include']['path'].append(inc_colon) if len(inc_eq) == 1: for inc_colon in inc_eq[0].split(':'): self._attrs['include']['path'].append(inc_colon) elif len(inc_eq) == 2: # TODO: support path for each file pass else: raise UserException('Wrong format include: %s' % inc) if opts.include_ini: process_include_option(opts.include_ini, self._attrs['include']) if opts.exclude_ini: process_exclude_option(opts.exclude_ini, self._attrs['exclude']) # parsing macro parameters if opts.macro: for line in opts.macro: for macro in line.split(','): macro_eq = macro.split('=') if len(macro_eq) == 1: self._attrs['include']['macro'][macro_eq[0]] = '1' elif len(macro_eq) == 2: self._attrs['include']['macro'][ macro_eq[0]] = macro_eq[1] else: raise UserException('Wrong format include: %s' % inc) files = None if opts.source: for line in opts.source: flags = OrderedDict() for subflag in line.lower().split(','): if subflag.find('=') > 0: key, value = subflag.split('=') if key == 'file': flags[key] = value.split(':') elif key == 'alias': p1, p2 = value.split(':') if p1.endswith('/'): p1 = p1[:-1] if p2.endswith('/'): p2 = p2[:-1] self._attrs['source']['alias'][p1] = p2 else: flags[key] = value else: flags[subflag] = None isfree = None isstrict = None if flags.has_key('format'): if flags['format'] == 'free': isfree = True elif flags['format'] == 'fixed': isfree = False else: raise UserException( 'format subflag of source flag should be either free or fixed.' ) if flags.has_key('strict'): if flags['strict'] == 'yes': isstrict = True elif flags['strict'] == 'no': isstrict = False else: raise UserException( 'strict subflag of source flag should be either yes or no.' ) if flags.has_key('file'): subflags = OrderedDict() if isfree: subflags['isfree'] = isfree if isstrict: subflags['isstrict'] = isstrict for file in flags['file']: abspath = os.path.abspath(file) if files is None: files = [] files.append(abspath) self._attrs['source']['file'][abspath] = subflags else: if isfree: self._attrs['source']['isfree'] = isfree if isstrict: self._attrs['source']['isstrict'] = isstrict # dupulicate paths per each alias if files is None: newpath = set() for path in self._attrs['include']['path']: newpath.add(path) for p1, p2 in self._attrs['source']['alias'].iteritems(): if path.startswith(p1): newpath.add(p2 + path[len(p1):]) elif path.startswith(p2): newpath.add(p1 + path[len(p2):]) self._attrs['include']['path'] = list(newpath) newfile = OrderedDict() for path, value in self._attrs['include']['file'].iteritems(): newfile[path] = value for p1, p2 in self._attrs['source']['alias'].iteritems(): if path.startswith(p1): newpath = p2 + path[len(p1):] newfile[newpath] = deepcopy(value) elif path.startswith(p2): newpath = p1 + path[len(p2):] newfile[newpath] = deepcopy(value) self._attrs['include']['file'] = newfile for path, value in self._attrs['include']['file'].iteritems(): if value.has_key('path'): newpath = set() for path in value['path']: newpath.add(path) for p1, p2 in self._attrs['source']['alias'].iteritems(): if path.startswith(p1): newpath.add(p2 + path[len(p1):]) elif path.startswith(p2): newpath.add(p1 + path[len(p2):]) value['path'] = list(newpath) # parsing debugging options if opts.debug: for dbg in opts.debug: param_path, value = dbg.split('=') param_split = param_path.lower().split('.') value_split = value.lower().split(',') curdict = self._attrs['debug'] for param in param_split[:-1]: curdict = curdict[param] exec('curdict[param_split[-1]] = value_split') # parsing logging options if opts.logging: for log in opts.logging: param_path, value = log.split('=') param_split = param_path.lower().split('.') value_split = value.lower().split(',') curdict = self._attrs['logging'] for param in param_split[:-1]: curdict = curdict[param] exec('curdict[param_split[-1]] = value_split') # mpi frame code in kernel driver if opts.add_mpi_frame: self._attrs['add_mpi_frame']['enabled'] = True for checkparams in opts.add_mpi_frame.split(','): key, value = checkparams.split('=') key = key.lower() if key in ['np', 'mpiexec']: self._attrs['add_mpi_frame'][key] = value else: print 'WARNING: %s is not supported add_mpi_frame parameter' % key if opts.outdir: self._attrs['path']['outdir'] = opts.outdir # create state directories and change working directory if not os.path.exists(self._attrs['path']['outdir']): os.makedirs(self._attrs['path']['outdir']) os.chdir(self._attrs['path']['outdir']) def __getattr__(self, name): return self._attrs[name]
class ConfigArgParser(argparse.ArgumentParser): """Drop-in replacement for argparse.ConfigArgParser that adds support for environment variables and .ini or .yaml-style config files. """ def __init__(self, prog=None, usage=None, description=None, epilog=None, version=None, parents=None, formatter_class=argparse.HelpFormatter, prefix_chars='-', fromfile_prefix_chars=None, argument_default=None, conflict_handler='error', add_help=True, add_config_file_help=True, add_env_var_help=True, auto_env_var_prefix=None, config_file_parser=None, default_config_files=None, ignore_unknown_config_file_keys=False, allow_unknown_config_file_keys=False, # deprecated args_for_setting_config_path=None, config_arg_is_required=False, config_arg_help_message="config file path", args_for_writing_out_config_file=None, write_out_config_file_arg_help_message="takes the current command line " "args and writes them out to a config file at the given path, then " "exits" ): # noinspection PyPep8 """Supports all the same args as the argparse.ConfigArgParser constructor, as well as the following additional args. Additional Args: add_config_file_help: Whether to add a description of config file syntax to the help message. add_env_var_help: Whether to add something to the help message for args that can be set through environment variables. auto_env_var_prefix: If set to a string instead of None, all config- file-settable options will become also settable via environment variables whose names are this prefix followed by the config file key, all in upper case. (eg. setting this to "foo_" will allow an arg like "--my-arg" to also be set via the FOO_MY_ARG environment variable) config_file_parser: An instance of a parser to be used for parsing config files. Default: ConfigFileParser() default_config_files: When specified, this list of config files will be parsed in order, with the values from each config file taking precedence over previous ones. This allows an application to look for config files in multiple standard locations such as the install directory, home directory, and current directory: ["<install dir>/app_config.ini", "~/.my_app_config.ini", "./app_config.txt"] ignore_unknown_config_file_keys: If true, settings that are found in a config file but don't correspond to any defined configargparse args will be ignored. If false, they will be processed and appended to the commandline like other args, and can be retrieved using parse_known_args() instead of parse_args() allow_unknown_config_file_keys: @deprecated Use ignore_unknown_config_file_keys instead. If true, settings that are found in a config file but don't correspond to any defined configargparse args, will still be processed and appended to the command line (eg. for parsing with parse_known_args()). If false, they will be ignored. args_for_setting_config_path: A list of one or more command line args to be used for specifying the config file path (eg. ["-c", "--config-file"]). Default: [] config_arg_is_required: When args_for_setting_config_path is set, set this to True to always require users to provide a config path. config_arg_help_message: the help message to use for the args listed in args_for_setting_config_path. args_for_writing_out_config_file: A list of one or more command line args to use for specifying a config file output path. If provided, these args cause configargparse to write out a config file with settings based on the other provided commandline args, environment variants and defaults, and then to exit. (eg. ["-w", "--write-out-config-file"]). Default: [] write_out_config_file_arg_help_message: The help message to use for the args in args_for_writing_out_config_file. """ if not parents: # noinspection PyPep8 parents = list() if not default_config_files: default_config_files = list() if not args_for_setting_config_path: args_for_setting_config_path = list() if not args_for_writing_out_config_file: args_for_writing_out_config_file = list() self._source_to_settings = None self._add_config_file_help = add_config_file_help self._add_env_var_help = add_env_var_help self._auto_env_var_prefix = auto_env_var_prefix # extract kwargs that can be passed to the super constructor # noinspection PyPep8 kwargs_for_super = dict( (k, v) for k, v in locals().items() if k in [ "prog", "usage", "description", "epilog", "version", "parents", "formatter_class", "prefix_chars", "fromfile_prefix_chars", "argument_default", "conflict_handler", "add_help"]) if sys.version_info >= (3, 3) and "version" in kwargs_for_super: # noinspection PyPep8 del kwargs_for_super[ "version"] # version arg deprecated in v3.3 argparse.ArgumentParser.__init__(self, **kwargs_for_super) # parse the additional args if config_file_parser is None: self._config_file_parser = ConfigFileParser() else: self._config_file_parser = config_file_parser self._default_config_files = default_config_files # noinspection PyPep8 self._ignore_unknown_config_file_keys = ignore_unknown_config_file_keys \ or allow_unknown_config_file_keys if args_for_setting_config_path: # noinspection PyPep8 self.add_argument(*args_for_setting_config_path, dest="config_file", required=config_arg_is_required, help=config_arg_help_message, is_config_file_arg=True) if args_for_writing_out_config_file: # noinspection PyPep8 self.add_argument(*args_for_writing_out_config_file, dest="write_out_config_file_to_this_path", metavar="CONFIG_OUTPUT_PATH", help=write_out_config_file_arg_help_message, is_write_out_config_file_arg=True) def parse_args(self, args=None, namespace=None, config_file_contents=None, env_vars=os.environ): # noinspection PyPep8 """Supports all the same args as the ConfigArgParser.parse_args(..), as well as the following additional args. Additional Args: args: a list of args as in argparse, or a string (eg. "-x -y bla") config_file_contents: String. Used for testing. env_vars: Dictionary. Used for testing. """ # noinspection PyPep8 args, argv = self.parse_known_args(args=args, namespace=namespace, config_file_contents=config_file_contents, env_vars=env_vars) if argv: self.error('unrecognized arguments: %s' % ' '.join(argv)) return args # noinspection PyPep8 def parse_known_args(self, args=None, namespace=None, config_file_contents=None, env_vars=os.environ): # noinspection PyPep8 """Supports all the same args as the ConfigArgParser.parse_args(..), as well as the following additional args. Additional Args: args: a list of args as in argparse, or a string (eg. "-x -y bla") config_file_contents: String. Used for testing. env_vars: Dictionary. Used for testing. """ if args is None: args = sys.argv[1:] elif type(args) == str: args = args.split() else: args = list(args) for a in self._actions: a.is_positional_arg = not a.option_strings # maps string describing the source (eg. env var) to a settings dict # to keep track of where values came from (used by print_values()) self._source_to_settings = OrderedDict() if args: # noinspection PyPep8 a_v_pair = ( None, list(args)) # copy args list to isolate changes # noinspection PyPep8 self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = { '': a_v_pair} # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed if self._auto_env_var_prefix is not None: for a in self._actions: config_file_keys = self.get_possible_config_keys(a) # noinspection PyPep8 if config_file_keys and not ( a.env_var or a.is_positional_arg or a.is_config_file_arg or a.is_write_out_config_file_arg): # noinspection PyPep8 stripped_config_file_key = config_file_keys[ 0].strip( self.prefix_chars) # noinspection PyPep8 a.env_var = (self._auto_env_var_prefix + stripped_config_file_key).replace('-', '_').upper() # add env var settings to the commandline that aren't there already env_var_args = [] # noinspection PyPep8 actions_with_env_var_values = [a for a in self._actions if not a.is_positional_arg and a.env_var and a.env_var in env_vars and not already_on_command_line( args, a.option_strings)] for action in actions_with_env_var_values: key = action.env_var value = env_vars[key] env_var_args += self.convert_setting_to_command_line_arg( action, key, value) args += env_var_args if env_var_args: self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict( [(a.env_var, (a, env_vars[a.env_var])) for a in actions_with_env_var_values]) # prepare for reading config file(s) # noinspection PyPep8 known_config_keys = dict( (config_key, action) for action in self._actions for config_key in self.get_possible_config_keys(action)) # open the config file(s) if config_file_contents: stream = StringIO(config_file_contents) stream.name = "method arg" config_streams = [stream] else: config_streams = self._open_config_files(args) config_settings = OrderedDict() # parse each config file # noinspection PyPep8 for stream in config_streams[::-1]: try: config_settings = self._config_file_parser.parse(stream) except ConfigFileParserException as e: self.error(e) finally: if hasattr(stream, "close"): stream.close() # add each config setting to the commandline unless it's there already config_args = [] for key, value in config_settings.items(): if key in known_config_keys: action = known_config_keys[key] discard_this_key = already_on_command_line( args, action.option_strings) else: action = None # noinspection PyPep8 discard_this_key = self._ignore_unknown_config_file_keys or \ already_on_command_line( args, self.get_command_line_key_for_unknown_config_file_setting( key)) if not discard_this_key: # noinspection PyPep8 config_args += self.convert_setting_to_command_line_arg( action, key, value) # noinspection PyPep8 source_key = "%s|%s" % ( _CONFIG_FILE_SOURCE_KEY, stream.name) if source_key not in self._source_to_settings: # noinspection PyPep8 self._source_to_settings[ source_key] = OrderedDict() # noinspection PyPep8 self._source_to_settings[source_key][key] = ( action, value) args += config_args # save default settings for use by print_values() default_settings = OrderedDict() for action in self._actions: # noinspection PyPep8 cares_about_default_value = (not action.is_positional_arg or action.nargs in [OPTIONAL, ZERO_OR_MORE]) # noinspection PyPep8 if (already_on_command_line(args, action.option_strings) or not cares_about_default_value or action.default is None or action.default == SUPPRESS or type( action) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE): continue else: if action.option_strings: key = action.option_strings[-1] else: key = action.dest default_settings[key] = (action, str(action.default)) if default_settings: # noinspection PyPep8 self._source_to_settings[ _DEFAULTS_SOURCE_KEY] = default_settings # parse all args (including commandline, config file, and env var) # noinspection PyPep8 namespace, unknown_args = argparse.ArgumentParser.parse_known_args( self, args=args, namespace=namespace) # handle any args that have is_write_out_config_file_arg set to true # noinspection PyPep8 user_write_out_config_file_arg_actions = [a for a in self._actions if getattr(a, "is_write_out_config_file_arg", False)] if user_write_out_config_file_arg_actions: output_file_paths = [] # noinspection PyPep8 for action in user_write_out_config_file_arg_actions: # check if the user specified this arg on the commandline output_file_path = getattr(namespace, action.dest, None) if output_file_path: # validate the output file path try: # noinspection PyUnusedLocal with open(output_file_path, "w") as output_file: output_file_paths.append(output_file_path) except IOError as e: # noinspection PyPep8 raise ValueError( "Couldn't open %s for writing: %s" % ( output_file_path, e)) if output_file_paths: # generate the config file contents config_items = self.get_items_for_config_file_output( self._source_to_settings, namespace) # noinspection PyPep8 contents = self._config_file_parser.serialize( config_items) for output_file_path in output_file_paths: with open(output_file_path, "w") as output_file: output_file.write(contents) if len(output_file_paths) == 1: output_file_paths = output_file_paths[0] # noinspection PyPep8 self.exit(message="Wrote config file to " + str( output_file_paths)) return namespace, unknown_args def get_command_line_key_for_unknown_config_file_setting(self, key): # noinspection PyPep8 """Compute a commandline arg key to be used for a config file setting that doesn't correspond to any defined configargparse arg (and so doesn't have a user-specified commandline arg key). Args: key: The config file key that was being set. """ key_without_prefix_chars = key.strip(self.prefix_chars) # noinspection PyPep8 command_line_key = self.prefix_chars[ 0] * 2 + key_without_prefix_chars return command_line_key def get_items_for_config_file_output(self, source_to_settings, parsed_namespace): # noinspection PyPep8 """Does the inverse of config parsing by taking parsed values and converting them back to a string representing config file contents. Args: source_to_settings: the dictionary created within parse_known_args() parsed_namespace: namespace object created within parse_known_args() Returns: an OrderedDict with the items to be written to the config file """ config_file_items = OrderedDict() for source, settings in source_to_settings.items(): if source == _COMMAND_LINE_SOURCE_KEY: _, existing_command_line_args = settings[''] for action in self._actions: # noinspection PyPep8 config_file_keys = self.get_possible_config_keys( action) # noinspection PyPep8 if config_file_keys and not action.is_positional_arg and \ already_on_command_line( existing_command_line_args, action.option_strings): # noinspection PyPep8 value = getattr(parsed_namespace, action.dest, None) if value is not None: if type(value) is bool: value = str(value).lower() elif type(value) is list: # noinspection PyPep8 value = "[" + ", ".join( map(str, value)) + "]" # noinspection PyPep8 config_file_items[ config_file_keys[0]] = value elif source == _ENV_VAR_SOURCE_KEY: for key, (action, value) in settings.items(): # noinspection PyPep8 config_file_keys = self.get_possible_config_keys( action) if config_file_keys: # noinspection PyPep8 value = getattr(parsed_namespace, action.dest, None) if value is not None: # noinspection PyPep8 config_file_items[ config_file_keys[0]] = value elif source.startswith(_CONFIG_FILE_SOURCE_KEY): for key, (action, value) in settings.items(): config_file_items[key] = value elif source == _DEFAULTS_SOURCE_KEY: for key, (action, value) in settings.items(): # noinspection PyPep8 config_file_keys = self.get_possible_config_keys( action) if config_file_keys: # noinspection PyPep8 value = getattr(parsed_namespace, action.dest, None) if value is not None: # noinspection PyPep8 config_file_items[ config_file_keys[0]] = value return config_file_items def convert_setting_to_command_line_arg(self, action, key, value): # noinspection PyPep8 """Converts a config file or env var key/value to a list of commandline args to append to the commandline. Args: action: The action corresponding to this setting, or None if this is a config file setting that doesn't correspond to any defined configargparse arg. key: The config file key or env var name value: The raw value string from the config file or env var """ if type(value) != str: raise ValueError("type(value) != str: %s" % str(value)) args = [] if action is None: # noinspection PyPep8 command_line_key = \ self.get_command_line_key_for_unknown_config_file_setting( key) else: command_line_key = action.option_strings[-1] if value.lower() == "true": if action is not None: # noinspection PyPep8 if type( action) not in ACTION_TYPES_THAT_DONT_NEED_A_VALUE: # noinspection PyPep8 self.error( "%s set to 'True' rather than a value" % key) # noinspection PyPep8 args.append(command_line_key) elif value.startswith("[") and value.endswith("]"): if action is not None: # noinspection PyProtectedMember if type(action) != argparse._AppendAction: # noinspection PyPep8 self.error( ("%s can't be set to a list '%s' unless its " "action type is changed to 'append'") % ( key, value)) for list_elem in value[1:-1].split(","): # noinspection PyPep8 args.append(command_line_key) # noinspection PyPep8 args.append(list_elem.strip()) else: if action is not None: if type(action) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE: # noinspection PyPep8 self.error( "%s is a flag but is being set to '%s'" % ( key, value)) # noinspection PyPep8 args.append(command_line_key) # noinspection PyPep8 args.append(value) return args def get_possible_config_keys(self, action): # noinspection PyPep8 """This method decides which actions can be set in a config file and what their keys will be. It returns a list of 0 or more config keys that can be used to set the given action's value in a config file. """ keys = [] for arg in action.option_strings: if any([arg.startswith(2 * c) for c in self.prefix_chars]): # noinspection PyPep8 keys += [arg[2:], arg] # eg. for '--bla' return ['bla', '--bla'] return keys # noinspection PyPep8 def _open_config_files(self, command_line_args): # noinspection PyPep8 """Tries to parse config file path(s) from within command_line_args. Returns a list of opened config files, including files specified on the commandline as well as any default_config_files specified in the constructor that are present on disk. Args: command_line_args: List of all args (already split on spaces) """ # open any default config files # noinspection PyPep8 config_files = [open(f) for f in map( os.path.expanduser, self._default_config_files) if os.path.isfile(f)] if not command_line_args: return config_files # list actions with is_config_file_arg=True. Its possible there is more # than one such arg. # noinspection PyPep8 user_config_file_arg_actions = [ a for a in self._actions if getattr(a, "is_config_file_arg", False)] if not user_config_file_arg_actions: return config_files # noinspection PyPep8 for action in user_config_file_arg_actions: # try to parse out the config file path by using a clean new # ConfigArgParser that only knows this one arg/action. arg_parser = argparse.ArgumentParser( prefix_chars=self.prefix_chars, add_help=False) arg_parser._add_action(action) # make parser not exit on error by replacing its error method. # Otherwise it sys.exits(..) if, for example, config file # is_required=True and user doesn't provide it. # noinspection PyUnusedLocal def error_method(message): """ :param message: :return: """ pass # noinspection PyArgumentList,PyPep8 arg_parser.error = types.MethodType(error_method, arg_parser) # check whether the user provided a value # noinspection PyPep8 parsed_arg = arg_parser.parse_known_args( args=command_line_args) if not parsed_arg: continue namespace, _ = parsed_arg user_config_file = getattr(namespace, action.dest, None) if not user_config_file: continue # validate the user-provided config file path user_config_file = os.path.expanduser(user_config_file) if not os.path.isfile(user_config_file): self.error('File not found: %s' % user_config_file) config_files += [open(user_config_file)] return config_files def format_values(self): """Returns a string with all args and settings and where they came from (eg. commandline, config file, environment variable or default) """ source_key_to_display_value_map = { _COMMAND_LINE_SOURCE_KEY: "Command Line Args: ", _ENV_VAR_SOURCE_KEY: "Environment Variables:\n", _CONFIG_FILE_SOURCE_KEY: "Config File (%s):\n", _DEFAULTS_SOURCE_KEY: "Defaults:\n" } r = StringIO() for source, settings in self._source_to_settings.items(): source = source.split("|") # noinspection PyPep8 source = source_key_to_display_value_map[source[0]] % tuple( source[1:]) r.write(source) for key, (action, value) in settings.items(): if key: r.write(" %-19s%s\n" % (key + ":", value)) else: if type(value) is str: r.write(" %s\n" % value) elif type(value) is list: r.write(" %s\n" % ' '.join(value)) return r.getvalue() def print_values(self, file=sys.stdout): # noinspection PyPep8 """Prints the format_values() string (to sys.stdout or another file).""" file.write(self.format_values()) def format_help(self): """ :return: """ msg = "" added_config_file_help = False added_env_var_help = False if self._add_config_file_help: default_config_files = self._default_config_files cc = 2 * self.prefix_chars[0] # eg. -- # noinspection PyPep8 config_settable_args = [(arg, a) for a in self._actions for arg in a.option_strings if self.get_possible_config_keys( a) and not ( a.dest == "help" or a.is_config_file_arg or a.is_write_out_config_file_arg)] # noinspection PyPep8 config_path_actions = [a for a in self._actions if getattr(a, "is_config_file_arg", False)] if config_settable_args and (default_config_files or config_path_actions): # noinspection PyPep8 self._add_config_file_help = False # prevent duplication added_config_file_help = True # noinspection PyPep8 msg += ( "Args that start with '%s' (eg. %s) can also be set in " "a config file") % ( cc, config_settable_args[0][0]) # noinspection PyPep8 config_arg_string = " or ".join(a.option_strings[0] for a in config_path_actions if a.option_strings) if config_arg_string: # noinspection PyPep8 config_arg_string = "specified via " + config_arg_string if default_config_files or config_arg_string: # noinspection PyPep8 msg += " (%s)." % " or ".join(default_config_files + [config_arg_string]) # noinspection PyPep8 msg += " " + self._config_file_parser.get_syntax_description() if self._add_env_var_help: env_var_actions = [(a.env_var, a) for a in self._actions if getattr(a, "env_var", None)] for env_var, a in env_var_actions: env_var_help_string = " [env var: %s]" % env_var if not a.help: a.help = "" if env_var_help_string not in a.help: a.help += env_var_help_string added_env_var_help = True # noinspection PyPep8 self._add_env_var_help = False # prevent duplication if added_env_var_help or added_config_file_help: value_sources = ["defaults"] if added_config_file_help: value_sources = ["config file values"] + value_sources if added_env_var_help: # noinspection PyPep8 value_sources = [ "environment variables"] + value_sources # noinspection PyPep8 msg += ( " If an arg is specified in more than one place, then " "commandline values override %s.") % ( " which override ".join(value_sources)) if msg: self.description = (self.description or "") + " " + msg return argparse.ArgumentParser.format_help(self)
def parse_known_args(self, args=None, namespace=None, config_file_contents=None, env_vars=os.environ): # noinspection PyPep8 """Supports all the same args as the ConfigArgParser.parse_args(..), as well as the following additional args. Additional Args: args: a list of args as in argparse, or a string (eg. "-x -y bla") config_file_contents: String. Used for testing. env_vars: Dictionary. Used for testing. """ if args is None: args = sys.argv[1:] elif type(args) == str: args = args.split() else: args = list(args) for a in self._actions: a.is_positional_arg = not a.option_strings # maps string describing the source (eg. env var) to a settings dict # to keep track of where values came from (used by print_values()) self._source_to_settings = OrderedDict() if args: # noinspection PyPep8 a_v_pair = ( None, list(args)) # copy args list to isolate changes # noinspection PyPep8 self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = { '': a_v_pair} # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed if self._auto_env_var_prefix is not None: for a in self._actions: config_file_keys = self.get_possible_config_keys(a) # noinspection PyPep8 if config_file_keys and not ( a.env_var or a.is_positional_arg or a.is_config_file_arg or a.is_write_out_config_file_arg): # noinspection PyPep8 stripped_config_file_key = config_file_keys[ 0].strip( self.prefix_chars) # noinspection PyPep8 a.env_var = (self._auto_env_var_prefix + stripped_config_file_key).replace('-', '_').upper() # add env var settings to the commandline that aren't there already env_var_args = [] # noinspection PyPep8 actions_with_env_var_values = [a for a in self._actions if not a.is_positional_arg and a.env_var and a.env_var in env_vars and not already_on_command_line( args, a.option_strings)] for action in actions_with_env_var_values: key = action.env_var value = env_vars[key] env_var_args += self.convert_setting_to_command_line_arg( action, key, value) args += env_var_args if env_var_args: self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict( [(a.env_var, (a, env_vars[a.env_var])) for a in actions_with_env_var_values]) # prepare for reading config file(s) # noinspection PyPep8 known_config_keys = dict( (config_key, action) for action in self._actions for config_key in self.get_possible_config_keys(action)) # open the config file(s) if config_file_contents: stream = StringIO(config_file_contents) stream.name = "method arg" config_streams = [stream] else: config_streams = self._open_config_files(args) config_settings = OrderedDict() # parse each config file # noinspection PyPep8 for stream in config_streams[::-1]: try: config_settings = self._config_file_parser.parse(stream) except ConfigFileParserException as e: self.error(e) finally: if hasattr(stream, "close"): stream.close() # add each config setting to the commandline unless it's there already config_args = [] for key, value in config_settings.items(): if key in known_config_keys: action = known_config_keys[key] discard_this_key = already_on_command_line( args, action.option_strings) else: action = None # noinspection PyPep8 discard_this_key = self._ignore_unknown_config_file_keys or \ already_on_command_line( args, self.get_command_line_key_for_unknown_config_file_setting( key)) if not discard_this_key: # noinspection PyPep8 config_args += self.convert_setting_to_command_line_arg( action, key, value) # noinspection PyPep8 source_key = "%s|%s" % ( _CONFIG_FILE_SOURCE_KEY, stream.name) if source_key not in self._source_to_settings: # noinspection PyPep8 self._source_to_settings[ source_key] = OrderedDict() # noinspection PyPep8 self._source_to_settings[source_key][key] = ( action, value) args += config_args # save default settings for use by print_values() default_settings = OrderedDict() for action in self._actions: # noinspection PyPep8 cares_about_default_value = (not action.is_positional_arg or action.nargs in [OPTIONAL, ZERO_OR_MORE]) # noinspection PyPep8 if (already_on_command_line(args, action.option_strings) or not cares_about_default_value or action.default is None or action.default == SUPPRESS or type( action) in ACTION_TYPES_THAT_DONT_NEED_A_VALUE): continue else: if action.option_strings: key = action.option_strings[-1] else: key = action.dest default_settings[key] = (action, str(action.default)) if default_settings: # noinspection PyPep8 self._source_to_settings[ _DEFAULTS_SOURCE_KEY] = default_settings # parse all args (including commandline, config file, and env var) # noinspection PyPep8 namespace, unknown_args = argparse.ArgumentParser.parse_known_args( self, args=args, namespace=namespace) # handle any args that have is_write_out_config_file_arg set to true # noinspection PyPep8 user_write_out_config_file_arg_actions = [a for a in self._actions if getattr(a, "is_write_out_config_file_arg", False)] if user_write_out_config_file_arg_actions: output_file_paths = [] # noinspection PyPep8 for action in user_write_out_config_file_arg_actions: # check if the user specified this arg on the commandline output_file_path = getattr(namespace, action.dest, None) if output_file_path: # validate the output file path try: # noinspection PyUnusedLocal with open(output_file_path, "w") as output_file: output_file_paths.append(output_file_path) except IOError as e: # noinspection PyPep8 raise ValueError( "Couldn't open %s for writing: %s" % ( output_file_path, e)) if output_file_paths: # generate the config file contents config_items = self.get_items_for_config_file_output( self._source_to_settings, namespace) # noinspection PyPep8 contents = self._config_file_parser.serialize( config_items) for output_file_path in output_file_paths: with open(output_file_path, "w") as output_file: output_file.write(contents) if len(output_file_paths) == 1: output_file_paths = output_file_paths[0] # noinspection PyPep8 self.exit(message="Wrote config file to " + str( output_file_paths)) return namespace, unknown_args
class ArgumentParser(argparse.ArgumentParser): """Drop-in replacement for argparse.ArgumentParser that adds support for environment variables and .ini or .yaml-style config files. """ def __init__( self, add_config_file_help=True, add_env_var_help=True, auto_env_var_prefix=None, default_config_files=[], ignore_unknown_config_file_keys=False, config_file_parser_class=DefaultConfigFileParser, args_for_setting_config_path=[], config_arg_is_required=False, config_arg_help_message="config file path", args_for_writing_out_config_file=[], write_out_config_file_arg_help_message="takes the current command line " "args and writes them out to a config file at the given path, then " "exits", **kwargs): """Supports args of the argparse.ArgumentParser constructor as **kwargs, as well as the following additional args. Additional Args: add_config_file_help: Whether to add a description of config file syntax to the help message. add_env_var_help: Whether to add something to the help message for args that can be set through environment variables. auto_env_var_prefix: If set to a string instead of None, all config- file-settable options will become also settable via environment variables whose names are this prefix followed by the config file key, all in upper case. (eg. setting this to "foo_" will allow an arg like "--my-arg" to also be set via the FOO_MY_ARG environment variable) default_config_files: When specified, this list of config files will be parsed in order, with the values from each config file taking precedence over pervious ones. This allows an application to look for config files in multiple standard locations such as the install directory, home directory, and current directory. Also, shell * syntax can be used to specify all conf files in a directory. For exmaple: ["/etc/conf/app_config.ini", "/etc/conf/conf-enabled/*.ini", "~/.my_app_config.ini", "./app_config.txt"] ignore_unknown_config_file_keys: If true, settings that are found in a config file but don't correspond to any defined configargparse args will be ignored. If false, they will be processed and appended to the commandline like other args, and can be retrieved using parse_known_args() instead of parse_args() config_file_parser_class: configargparse.ConfigFileParser subclass which determines the config file format. configargparse comes with DefaultConfigFileParser and YAMLConfigFileParser. args_for_setting_config_path: A list of one or more command line args to be used for specifying the config file path (eg. ["-c", "--config-file"]). Default: [] config_arg_is_required: When args_for_setting_config_path is set, set this to True to always require users to provide a config path. config_arg_help_message: the help message to use for the args listed in args_for_setting_config_path. args_for_writing_out_config_file: A list of one or more command line args to use for specifying a config file output path. If provided, these args cause configargparse to write out a config file with settings based on the other provided commandline args, environment variants and defaults, and then to exit. (eg. ["-w", "--write-out-config-file"]). Default: [] write_out_config_file_arg_help_message: The help message to use for the args in args_for_writing_out_config_file. """ self._add_config_file_help = add_config_file_help self._add_env_var_help = add_env_var_help self._auto_env_var_prefix = auto_env_var_prefix argparse.ArgumentParser.__init__(self, **kwargs) # parse the additional args if config_file_parser_class is None: self._config_file_parser = DefaultConfigFileParser() else: self._config_file_parser = config_file_parser_class() self._default_config_files = default_config_files self._ignore_unknown_config_file_keys = ignore_unknown_config_file_keys if args_for_setting_config_path: self.add_argument(*args_for_setting_config_path, dest="config_file", required=config_arg_is_required, help=config_arg_help_message, is_config_file_arg=True) if args_for_writing_out_config_file: self.add_argument(*args_for_writing_out_config_file, dest="write_out_config_file_to_this_path", metavar="CONFIG_OUTPUT_PATH", help=write_out_config_file_arg_help_message, is_write_out_config_file_arg=True) def parse_args(self, args=None, namespace=None, config_file_contents=None, env_vars=os.environ): """Supports all the same args as the ArgumentParser.parse_args(..), as well as the following additional args. Additional Args: args: a list of args as in argparse, or a string (eg. "-x -y bla") config_file_contents: String. Used for testing. env_vars: Dictionary. Used for testing. """ args, argv = self.parse_known_args( args=args, namespace=namespace, config_file_contents=config_file_contents, env_vars=env_vars) if argv: self.error('unrecognized arguments: %s' % ' '.join(argv)) return args def parse_known_args(self, args=None, namespace=None, config_file_contents=None, env_vars=os.environ): """Supports all the same args as the ArgumentParser.parse_args(..), as well as the following additional args. Additional Args: args: a list of args as in argparse, or a string (eg. "-x -y bla") config_file_contents: String. Used for testing. env_vars: Dictionary. Used for testing. """ if args is None: args = sys.argv[1:] elif isinstance(args, str): args = args.split() else: args = list(args) # normalize args by converting args like --key=value to --key value normalized_args = list() for arg in args: if arg and arg[0] in self.prefix_chars and '=' in arg: key, value = arg.split('=', 1) normalized_args.append(key) normalized_args.append(value) else: normalized_args.append(arg) args = normalized_args for a in self._actions: a.is_positional_arg = not a.option_strings # maps a string describing the source (eg. env var) to a settings dict # to keep track of where values came from (used by print_values()). # The settings dicts for env vars and config files will then map # the config key to an (argparse Action obj, string value) 2-tuple. self._source_to_settings = OrderedDict() if args: a_v_pair = (None, list(args)) # copy args list to isolate changes self._source_to_settings[_COMMAND_LINE_SOURCE_KEY] = {'': a_v_pair} # handle auto_env_var_prefix __init__ arg by setting a.env_var as needed if self._auto_env_var_prefix is not None: for a in self._actions: config_file_keys = self.get_possible_config_keys(a) if config_file_keys and not ( a.env_var or a.is_positional_arg or a.is_config_file_arg or a.is_write_out_config_file_arg or isinstance(a, argparse._HelpAction)): stripped_config_file_key = config_file_keys[0].strip( self.prefix_chars) a.env_var = (self._auto_env_var_prefix + stripped_config_file_key).replace( '-', '_').upper() # add env var settings to the commandline that aren't there already env_var_args = [] actions_with_env_var_values = [ a for a in self._actions if not a.is_positional_arg and a.env_var and a.env_var in env_vars and not already_on_command_line(args, a.option_strings) ] for action in actions_with_env_var_values: key = action.env_var value = env_vars[key] # Make list-string into list. if action.nargs or isinstance(action, argparse._AppendAction): element_capture = re.match('\[(.*)\]', value) if element_capture: value = [ val.strip() for val in element_capture.group(1).split(',') if val.strip() ] env_var_args += self.convert_item_to_command_line_arg( action, key, value) args = args + env_var_args if env_var_args: self._source_to_settings[_ENV_VAR_SOURCE_KEY] = OrderedDict([ (a.env_var, (a, env_vars[a.env_var])) for a in actions_with_env_var_values ]) # before parsing any config files, check if -h was specified. supports_help_arg = any(a for a in self._actions if isinstance(a, argparse._HelpAction)) skip_config_file_parsing = supports_help_arg and ("-h" in args or "--help" in args) # prepare for reading config file(s) known_config_keys = dict( (config_key, action) for action in self._actions for config_key in self.get_possible_config_keys(action)) # open the config file(s) config_streams = [] if config_file_contents is not None: stream = StringIO(config_file_contents) stream.name = "method arg" config_streams = [stream] elif not skip_config_file_parsing: config_streams = self._open_config_files(args) # parse each config file for stream in reversed(config_streams): try: config_items = self._config_file_parser.parse(stream) except ConfigFileParserException as e: self.error(e) finally: if hasattr(stream, "close"): stream.close() # add each config item to the commandline unless it's there already config_args = [] for key, value in config_items.items(): if key in known_config_keys: action = known_config_keys[key] discard_this_key = already_on_command_line( args, action.option_strings) else: action = None discard_this_key = self._ignore_unknown_config_file_keys or \ already_on_command_line( args, [self.get_command_line_key_for_unknown_config_file_setting(key)]) if not discard_this_key: config_args += self.convert_item_to_command_line_arg( action, key, value) source_key = "%s|%s" % (_CONFIG_FILE_SOURCE_KEY, stream.name) if source_key not in self._source_to_settings: self._source_to_settings[source_key] = OrderedDict() self._source_to_settings[source_key][key] = (action, value) args = args + config_args # save default settings for use by print_values() default_settings = OrderedDict() for action in self._actions: cares_about_default_value = (not action.is_positional_arg or action.nargs in [OPTIONAL, ZERO_OR_MORE]) if (already_on_command_line(args, action.option_strings) or not cares_about_default_value or action.default is None or action.default == SUPPRESS or isinstance( action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE)): continue else: if action.option_strings: key = action.option_strings[-1] else: key = action.dest default_settings[key] = (action, str(action.default)) if default_settings: self._source_to_settings[_DEFAULTS_SOURCE_KEY] = default_settings # parse all args (including commandline, config file, and env var) namespace, unknown_args = argparse.ArgumentParser.parse_known_args( self, args=args, namespace=namespace) # handle any args that have is_write_out_config_file_arg set to true # check if the user specified this arg on the commandline output_file_paths = [ getattr(namespace, a.dest, None) for a in self._actions if getattr(a, "is_write_out_config_file_arg", False) ] output_file_paths = [a for a in output_file_paths if a is not None] self.write_config_file(namespace, output_file_paths, exit_after=True) return namespace, unknown_args def write_config_file(self, parsed_namespace, output_file_paths, exit_after=False): """Write the given settings to output files. Args: parsed_namespace: namespace object created within parse_known_args() output_file_paths: any number of file paths to write the config to exit_after: whether to exit the program after writing the config files """ for output_file_path in output_file_paths: # validate the output file path try: with open(output_file_path, "w") as output_file: pass except IOError as e: raise ValueError("Couldn't open %s for writing: %s" % (output_file_path, e)) if output_file_paths: # generate the config file contents config_items = self.get_items_for_config_file_output( self._source_to_settings, parsed_namespace) file_contents = self._config_file_parser.serialize(config_items) for output_file_path in output_file_paths: with open(output_file_path, "w") as output_file: output_file.write(file_contents) message = "Wrote config file to " + ", ".join(output_file_paths) if exit_after: self.exit(0, message) else: print(message) def get_command_line_key_for_unknown_config_file_setting(self, key): """Compute a commandline arg key to be used for a config file setting that doesn't correspond to any defined configargparse arg (and so doesn't have a user-specified commandline arg key). Args: key: The config file key that was being set. """ key_without_prefix_chars = key.strip(self.prefix_chars) command_line_key = self.prefix_chars[0] * 2 + key_without_prefix_chars return command_line_key def get_items_for_config_file_output(self, source_to_settings, parsed_namespace): """Converts the given settings back to a dictionary that can be passed to ConfigFormatParser.serialize(..). Args: source_to_settings: the dictionary described in parse_known_args() parsed_namespace: namespace object created within parse_known_args() Returns: an OrderedDict where keys are strings and values are either strings or lists """ config_file_items = OrderedDict() for source, settings in source_to_settings.items(): if source == _COMMAND_LINE_SOURCE_KEY: _, existing_command_line_args = settings[''] for action in self._actions: config_file_keys = self.get_possible_config_keys(action) if config_file_keys and not action.is_positional_arg and \ already_on_command_line(existing_command_line_args, action.option_strings): value = getattr(parsed_namespace, action.dest, None) if value is not None: if isinstance(value, bool): value = str(value).lower() elif callable(action.type): found = [ i for i in range( 0, len(existing_command_line_args) - 1) if existing_command_line_args[i] in config_file_keys ] if found: value = existing_command_line_args[ found[-1] + 1] config_file_items[config_file_keys[0]] = value elif source == _ENV_VAR_SOURCE_KEY: for key, (action, value) in settings.items(): config_file_keys = self.get_possible_config_keys(action) if config_file_keys: value = getattr(parsed_namespace, action.dest, None) if value is not None: config_file_items[config_file_keys[0]] = value elif source.startswith(_CONFIG_FILE_SOURCE_KEY): for key, (action, value) in settings.items(): config_file_items[key] = value elif source == _DEFAULTS_SOURCE_KEY: for key, (action, value) in settings.items(): config_file_keys = self.get_possible_config_keys(action) if config_file_keys: value = getattr(parsed_namespace, action.dest, None) if value is not None: config_file_items[config_file_keys[0]] = value return config_file_items def convert_item_to_command_line_arg(self, action, key, value): """Converts a config file or env var key + value to a list of commandline args to append to the commandline. Args: action: The argparse Action object for this setting, or None if this config file setting doesn't correspond to any defined configargparse arg. key: string (config file key or env var name) value: parsed value of type string or list """ args = [] if action is None: command_line_key = \ self.get_command_line_key_for_unknown_config_file_setting(key) else: command_line_key = action.option_strings[-1] # handle boolean value if action is not None and isinstance( action, ACTION_TYPES_THAT_DONT_NEED_A_VALUE): if value.lower() in ("true", "yes", "1"): args.append(command_line_key) elif value.lower() in ("false", "no", "0"): # don't append when set to "false" / "no" pass else: self.error("Unexpected value for %s: '%s'. Expecting 'true', " "'false', 'yes', 'no', '1' or '0'" % (key, value)) elif isinstance(value, list): if action is None or isinstance(action, argparse._AppendAction): for list_elem in value: args.append(command_line_key) args.append(str(list_elem)) elif (isinstance(action, argparse._StoreAction) and action.nargs in ('+', '*')) or (isinstance( action.nargs, int) and action.nargs > 1): args.append(command_line_key) for list_elem in value: args.append(str(list_elem)) else: self.error(( "%s can't be set to a list '%s' unless its action type is changed " "to 'append' or nargs is set to '*', '+', or > 1") % (key, value)) elif isinstance(value, str): args.append(command_line_key) args.append(value) else: raise ValueError("Unexpected value type %s for value: %s" % (type(value), value)) return args def get_possible_config_keys(self, action): """This method decides which actions can be set in a config file and what their keys will be. It returns a list of 0 or more config keys that can be used to set the given action's value in a config file. """ keys = [] # Do not write out the config options for writing out a config file if getattr(action, 'is_write_out_config_file_arg', None): return keys for arg in action.option_strings: if any([arg.startswith(2 * c) for c in self.prefix_chars]): keys += [arg[2:], arg] # eg. for '--bla' return ['bla', '--bla'] return keys def _open_config_files(self, command_line_args): """Tries to parse config file path(s) from within command_line_args. Returns a list of opened config files, including files specified on the commandline as well as any default_config_files specified in the constructor that are present on disk. Args: command_line_args: List of all args (already split on spaces) """ # open any default config files config_files = [ open(f) for files in map( glob.glob, map(os.path.expanduser, self._default_config_files)) for f in files ] # list actions with is_config_file_arg=True. Its possible there is more # than one such arg. user_config_file_arg_actions = [ a for a in self._actions if getattr(a, "is_config_file_arg", False) ] if not user_config_file_arg_actions: return config_files for action in user_config_file_arg_actions: # try to parse out the config file path by using a clean new # ArgumentParser that only knows this one arg/action. arg_parser = argparse.ArgumentParser( prefix_chars=self.prefix_chars, add_help=False) arg_parser._add_action(action) # make parser not exit on error by replacing its error method. # Otherwise it sys.exits(..) if, for example, config file # is_required=True and user doesn't provide it. def error_method(self, message): pass arg_parser.error = types.MethodType(error_method, arg_parser) # check whether the user provided a value parsed_arg = arg_parser.parse_known_args(args=command_line_args) if not parsed_arg: continue namespace, _ = parsed_arg user_config_file = getattr(namespace, action.dest, None) if not user_config_file: continue # validate the user-provided config file path user_config_file = os.path.expanduser(user_config_file) if not os.path.isfile(user_config_file): self.error('File not found: %s' % user_config_file) config_files += [open(user_config_file)] return config_files def format_values(self): """Returns a string with all args and settings and where they came from (eg. commandline, config file, enviroment variable or default) """ source_key_to_display_value_map = { _COMMAND_LINE_SOURCE_KEY: "Command Line Args: ", _ENV_VAR_SOURCE_KEY: "Environment Variables:\n", _CONFIG_FILE_SOURCE_KEY: "Config File (%s):\n", _DEFAULTS_SOURCE_KEY: "Defaults:\n" } r = StringIO() for source, settings in self._source_to_settings.items(): source = source.split("|") source = source_key_to_display_value_map[source[0]] % tuple( source[1:]) r.write(source) for key, (action, value) in settings.items(): if key: r.write(" %-19s%s\n" % (key + ":", value)) else: if isinstance(value, str): r.write(" %s\n" % value) elif isinstance(value, list): r.write(" %s\n" % ' '.join(value)) return r.getvalue() def print_values(self, file=sys.stdout): """Prints the format_values() string (to sys.stdout or another file).""" file.write(self.format_values()) def format_help(self): msg = "" added_config_file_help = False added_env_var_help = False if self._add_config_file_help: default_config_files = self._default_config_files cc = 2 * self.prefix_chars[0] # eg. -- config_settable_args = [ (arg, a) for a in self._actions for arg in a.option_strings if self.get_possible_config_keys(a) and not (a.dest == "help" or a.is_config_file_arg or a.is_write_out_config_file_arg) ] config_path_actions = [ a for a in self._actions if getattr(a, "is_config_file_arg", False) ] if config_settable_args and (default_config_files or config_path_actions): self._add_config_file_help = False # prevent duplication added_config_file_help = True msg += ( "Args that start with '%s' (eg. %s) can also be set in " "a config file") % (cc, config_settable_args[0][0]) config_arg_string = " or ".join(a.option_strings[0] for a in config_path_actions if a.option_strings) if config_arg_string: config_arg_string = "specified via " + config_arg_string if default_config_files or config_arg_string: msg += " (%s)." % " or ".join( tuple(default_config_files) + tuple(filter(None, [config_arg_string]))) msg += " " + self._config_file_parser.get_syntax_description() if self._add_env_var_help: env_var_actions = [(a.env_var, a) for a in self._actions if getattr(a, "env_var", None)] for env_var, a in env_var_actions: env_var_help_string = " [env var: %s]" % env_var if not a.help: a.help = "" if env_var_help_string not in a.help: a.help += env_var_help_string added_env_var_help = True self._add_env_var_help = False # prevent duplication if added_env_var_help or added_config_file_help: value_sources = ["defaults"] if added_config_file_help: value_sources = ["config file values"] + value_sources if added_env_var_help: value_sources = ["environment variables"] + value_sources msg += (" If an arg is specified in more than one place, then " "commandline values override %s.") % ( " which override ".join(value_sources)) if msg: self.description = (self.description or "") + " " + msg return argparse.ArgumentParser.format_help(self)
('chr4', (0, 1351857)), ('chrX', (0, 22422827)), ('chr2LHet', (0, 368872)), ('chr2RHet', (0, 3288761)), ('chr3LHet', (0, 2555491)), ('chr3RHet', (0, 2517507)), ('chrM', (0, 19517)), ('chrU', (0, 10049037)), ('chrUextra', (0, 29004656)), ('chrXHet', (0, 204112)), ('chrYHet', (0, 347038)), )) # No chrUextra or chrM dm3.default = OrderedDict() for chrom, size in dm3.items(): if chrom in ['chrUextra', 'chrM']: continue dm3.default[chrom] = size # No chrU*, chr*Het, or chrM dm3.euchromatic = OrderedDict() for chrom, size in dm3.default.items(): if 'chrU' in chrom: continue if 'Het' in chrom: continue dm3.euchromatic[chrom] = size mm9 = OrderedDict(( ('chr1', (0, 197195432)),
def GenInteractions_int( G_system, g_bond_pattern, typepattern_to_coefftypes, canonical_order, # function to sort atoms and bonds atomtypes_int2str, bondtypes_int2str, report_progress=False, # print messages to sys.stderr? check_undefined_atomids_str=None): """ GenInteractions() automatically determines a list of interactions present in a system of bonded atoms (argument "G_system"), which satisfy the bond topology present in "g_bond_pattern", and satisfy the atom and bond type requirements in "typepattern_to_coefftypes". Whenever a set of atoms in "G_system" are bonded together in a way which matches "g_bond_pattern", and when the atom and bond types is consistent with one of the entries in "typepattern_to_coefftypes", the corresponding list of atoms from G_system is appended to the list of results. These results (the list of lists of atoms participating in an interaction) are organized according their corresponding "coefftype", a string which identifies the type of interaction they obey as explained above. results are returned as a dictionary using "coefftype" as the lookup key. Arguments: -- typepattern_to_coefftypes is a list of 2-tuples -- The first element of the 2-tuple is the "typepattern". It contains a string describing a list of atom types and bond types. The typepattern is associated with a "coefftype", which is the second element of the 2-tuple. This is a string which identifies the type of interaction between the atoms. Later on, this string can be used to lookup the force field parameters for this interaction elsewhere.) -- Arguments: G_system, g_bond_pattern, atomtypes_int2str, bondtypes_int2str -- G_system stores a list of atoms and bonds, and their attributes in "Ugraph" format. In this format: Atom ID numbers are represented by indices into the G_system.verts[] list. Bond ID numbers are represented by indices into the G_system.edges[] list. Atom types are represented as integers in the G_system.verts[i].attr list. Bond types are represented as integers in the G_system.edges[i].attr list. They are converted into strings using atomtypes_int2str, and bondtypes_int2str. g_bond_pattern is a graph which specifies the type of bonding between the atoms required for a match. It is in Ugraph format (however the atom and bond types are left blank.) Atom and bond types are supplied by the user in string format. (These strings typically encode integers, but could be any string in principle.) The string-version of the ith atom type is stored in atomtypes_int2str[ G_system.verts[i].attr ] The string-version of the ith bond type is stored in bondtypes_int2str[ G_system.edges[i].attr ] -- The "canonical_order" argument: -- The search for atoms with a given bond pattern often yields redundant matches. There is no difference for example between the angle formed between three consecutively bonded atoms (named, 1, 2, 3, for example), and the angle between the same atoms in reverse order (3, 2, 1). However both triplets of atoms will be returned by the subgraph- matching algorithm when searching for ALL 3-body interactions.) To eliminate this redundancy, the caller must supply a "canonical_order" argument. This is a function which sorts the atoms and bonds in a way which is consistent with the type of N-body interaction being considered. The atoms (and bonds) in a candidate match are rearranged by the canonical_order(). Then the re-ordered list of atom and bond ids is tested against the list of atom/bond ids in the matches-found-so-far, before it is added. """ if report_progress: startatomid = 0 sys.stderr.write(' searching for matching bond patterns:\n') sys.stderr.write(' 0%') # Figure out which atoms from "G_system" bond together in a way which # matches the "g_bond_pattern" argument. Organize these matches by # atom and bond types and store all of the non-redundant ones in # the "interactions_by_type" variable. gm = GraphMatcher(G_system, g_bond_pattern) interactions_by_type = defaultdict(list) for atombondids in gm.Matches(): # "atombondids" is a tuple. # atombondids[0] has atomIDs from G_system corresponding to g_bond_pattern # (These atomID numbers are indices into the G_system.verts[] list.) # atombondids[1] has bondIDs from G_system corresponding to g_bond_pattern # (These bondID numbers are indices into the G_system.edges[] list.) # It's convenient to organize the list of interactions-between- # atoms in a dictionary indexed by atomtypes and bondtypes. # (Because many atoms and bonds typically share the same type, # organizing the results this way makes it faster to check # whether a given interaction matches a "typepattern" defined # by the user. We only have to check once for the whole group.) atombondtypes = \ (tuple([G_system.GetVert(Iv).attr for Iv in atombondids[0]]), tuple([G_system.GetEdge(Ie).attr for Ie in atombondids[1]])) interactions_by_type[atombondtypes].append(atombondids) if report_progress: # GraphMatcher.Matches() searches for matches in an order # that selects a different atomid number from G_system, # starting at 0, and continuing up to the number of atoms (-1) # in the system (G_system.nv-1), and using this as the first # atom in the match (ie match[0][0]). This number can be used # to guess much progress has been made so far. oldatomid = startatomid startatomid = atombondids[0][0] percent_complete = (100 * startatomid) // G_system.GetNumVerts() # report less often as more progress made if percent_complete <= 4: old_pc = (100 * oldatomid) // G_system.GetNumVerts() if percent_complete > old_pc: sys.stderr.write(' ' + str(percent_complete) + '%') elif percent_complete <= 8: pc_d2 = (100 * startatomid) // (2 * G_system.GetNumVerts()) oldpc_d2 = (100 * oldatomid) // (2 * G_system.GetNumVerts()) if pc_d2 > oldpc_d2: sys.stderr.write(' ' + str(percent_complete) + '%') elif percent_complete <= 20: pc_d4 = (100 * startatomid) // (4 * G_system.GetNumVerts()) oldpc_d4 = (100 * oldatomid) // (4 * G_system.GetNumVerts()) if pc_d4 > oldpc_d4: sys.stderr.write(' ' + str(percent_complete) + '%') else: pc_d10 = (100 * startatomid) // (10 * G_system.GetNumVerts()) oldpc_d10 = (100 * oldatomid) // (10 * G_system.GetNumVerts()) if pc_d10 > oldpc_d10: sys.stderr.write(' ' + str(percent_complete) + '%') if report_progress: sys.stderr.write(' 100%\n') #sys.stderr.write(' ...done\n') #sys.stderr.write(' Looking up available atom and bond types...') #coefftype_to_atomids = defaultdict(list) #abids_to_coefftypes = defaultdict(list) coefftype_to_atomids = OrderedDict() abids_to_coefftypes = OrderedDict() # -------------------- reporting progress ----------------------- if report_progress: # The next interval of code is not technically necessary, but it makes # the printed output easier to read by excluding irrelevant interactions # Now, test each match to see if the atoms and bonds involved match # any of the type-patterns in the "typepattern_to_coefftypes" argument. types_atoms_all_str = set([]) types_bonds_all_str = set([]) for typepattern, coefftype in typepattern_to_coefftypes: for atombondtypes, abidslist in interactions_by_type.items(): for Iv in atombondtypes[0]: types_atoms_all_str.add(atomtypes_int2str[Iv]) for Ie in atombondtypes[1]: types_bonds_all_str.add(bondtypes_int2str[Ie]) # ------------------ reporting progress (end) ------------------- # ------------------ check to make sure all interactions are defined ------ if check_undefined_atomids_str: # Checking for missing interactions is a headache. # Please excuse the messy code below. atomids_matched = OrderedDict() # Then loop through all the interactions (tuples of atoms) found by # GraphMatcher, sort the atoms and store them in dictionary # (atomids_matched) which keeps track of which interactions have # been defined (ie have force-field parameters assigned to them). # Initialize them to False, and update as interactions are found. for atombondtypes, abidslist in interactions_by_type.items(): for abids in abidslist: abids = canonical_order(abids) atomids_int = tuple(abids[0]) # NOTE TO SELF: # If in the future, different interactions (type_patterns) have # different symmetries, and canonical_order() varies from # interaction to interaction, then DONT loop over type_pattern: # for type_pattern, coefftype in typepattern_to_coefftypes) # abids = canonical_order(abids, type_pattern) # Why: When checking for undefined interactions, # we just want to make sure that SOME kind of interaction # involving these atoms exists. The gruesome details of # force-field symmetry should not enter into this. # (We certainly don't want to require that different # interactions are simultaneously present for the same set of # atoms for ALL the possible different atom orderings for the # different possible symmetries in the force field you are using # Perhaps, in the future I should just use something like this: # atomids_int = abids[0] # atomids_int.sort() # atomids_int = tuple(atomids_int) # This would work for most molecules. # I suppose that in some some bizarre molecules containing # triangular or square cycles, for example, this would not # distinguish all 3 angles in the triangle, for example. # mistakenly thinking there was only one interaction there. # But these cases are rare.) if not atomids_int in atomids_matched: atomids_matched[atomids_int] = False # (Later on, we'll set some of these to True) # ------------------ check to make sure all interactions are defined (end) count = 0 for typepattern, coefftype in typepattern_to_coefftypes: # ------------------ reporting progress ----------------------- # The next interval of code is not technically necessary, but it makes # the printed output easier to read by excluding irrelevant # interactions if report_progress: # Check to see if the atoms or bonds referred to in typepattern # are (potentially) satisfied by any of the atoms present in the system. # If any of the required atoms for this typepattern are not present # in this system, then skip to the next typepattern. atoms_available_Iv = [ False for Iv in range(0, g_bond_pattern.GetNumVerts()) ] for Iv in range(0, g_bond_pattern.GetNumVerts()): for type_atom_str in types_atoms_all_str: if MatchesPattern(type_atom_str, typepattern[Iv]): atoms_available_Iv[Iv] = True atoms_available = True for Iv in range(0, g_bond_pattern.GetNumVerts()): if not atoms_available_Iv[Iv]: atoms_available = False bonds_available_Ie = [ False for Ie in range(0, g_bond_pattern.GetNumEdges()) ] for Ie in range(0, g_bond_pattern.GetNumEdges()): for type_bond_str in types_bonds_all_str: if MatchesPattern( type_bond_str, typepattern[g_bond_pattern.GetNumVerts() + Ie]): bonds_available_Ie[Ie] = True bonds_available = True for Ie in range(0, g_bond_pattern.GetNumEdges()): if not bonds_available_Ie[Ie]: bonds_available = False if atoms_available and bonds_available: # Explanation: # (Again) only if ALL of the atoms and bond requirements for # this type pattern are satisfied by at least SOME of the atoms # present in the this system, ...THEN print a status message. # (Because for complex all-atom force-fields, the number of # possible atom types, and typepatterns far exceeds the number # of atom types typically present in the system. Otherwise # hundreds of kB of irrelevant information can be printed.) sys.stderr.write(' checking ' + coefftype + ' type requirements:' #' (atom-types,bond-types) ' '\n ' + str(typepattern) + '\n') # ------------------ reporting progress (end) ------------------- for atombondtypes, abidslist in interactions_by_type.items(): # express atom & bond types in a tuple of the original string # format types_atoms = [atomtypes_int2str[Iv] for Iv in atombondtypes[0]] types_bonds = [bondtypes_int2str[Ie] for Ie in atombondtypes[1]] type_strings = types_atoms + types_bonds # use string comparisons to check for a match with typepattern if MatchesAll(type_strings, typepattern): # <-see "ttree_lex.py" for abids in abidslist: # Re-order the atoms (and bonds) in a "canonical" way. # Only add new interactions to the list after re-ordering # them and checking that they have not been added earlier. # (...well not when using the same coefftype at least. # This prevents the same triplet of atoms from # being used to calculate the bond-angle twice: # once for 1-2-3 and 3-2-1, for example.) abids = canonical_order(abids) redundant = False if abids in abids_to_coefftypes: coefftypes = abids_to_coefftypes[abids] if coefftype in coefftypes: redundant = True if check_undefined_atomids_str: atomids_int = tuple(abids[0]) atomids_matched[atomids_int] = True if not redundant: # (It's too bad python does not # have an Ordered defaultdict) if coefftype in coefftype_to_atomids: coefftype_to_atomids[coefftype].append(abids[0]) else: coefftype_to_atomids[coefftype] = [abids[0]] if abids in abids_to_coefftypes: abids_to_coefftypes[abids].append(coefftype) else: abids_to_coefftypes[abids] = [coefftype] count += 1 if report_progress: sys.stderr.write(' (found ' + str(count) + ' non-redundant matches)\n') if check_undefined_atomids_str: for atomids_int, found_match in atomids_matched.items(): if not found_match: atomids_str = [ check_undefined_atomids_str[Iv] for Iv in atomids_int ] raise InputError( 'Error: A bonded interaction should exist between atoms:\n' + ' ' + (',\n '.join(atomids_str)) + '\n' + ' ...however no interaction between these types of atoms has been defined\n' + ' This usually means that at least one of your atom TYPES is incorrect.\n' + ' If this is not the case, then you can override this error message by\n' + ' invoking moltemplate.sh without the \"-checkff\" argument.\n' ) return coefftype_to_atomids
class TextDisplayResponder(plugins.Responder): def __init__(self, optionMap, *args): plugins.Responder.__init__(self) self.enableColor = optionMap.has_key("zen") self.enableSummary = optionMap.has_key("b") and not optionMap.has_key( "s") and not optionMap.has_key("coll") self.failedTests = [] self.resultSummary = OrderedDict() self.resultSummary["Tests Run"] = 0 def getSummaryKey(self, category): if category == "success": return "" elif category == "bug": return "Known Bugs" elif category.startswith("faster") or category.startswith( "slower") or category == "smaller" or category == "larger": return "Performance Differences" elif category in ["killed", "unrunnable", "cancelled", "abandoned"]: return "Incomplete" else: return "Failures" def shouldDescribe(self, test): return test.state.hasFailed() def writeDescription(self, test, summary): if self.enableColor and test.state.hasFailed(): self.printTestWithColorEnabled(test, colorer.RED, summary) else: self.describe(test, summary) def notifyComplete(self, test): if self.enableSummary: self.resultSummary["Tests Run"] += 1 summaryKey = self.getSummaryKey(test.state.category) if summaryKey: self.failedTests.append(test) if summaryKey not in self.resultSummary: self.resultSummary[summaryKey] = 0 self.resultSummary[summaryKey] += 1 if self.shouldDescribe(test): self.writeDescription(test, summary=False) def notifyAllComplete(self): if self.enableSummary: plugins.log.info("Results:") plugins.log.info("") if len(self.failedTests): plugins.log.info("Tests that did not succeed:") for test in self.failedTests: self.writeDescription(test, summary=True) plugins.log.info("") parts = [ summaryKey + ": " + str(count) for summaryKey, count in self.resultSummary.items() ] plugins.log.info(", ".join(parts)) def printTestWithColorEnabled(self, test, color, summary): colorer.enableOutputColor(color) self.describe(test, summary) colorer.disableOutputColor() def getPrefix(self, test): return test.getIndent() def describe(self, test, summary): prefix = " " if summary else self.getPrefix(test) desc = test.state.description() if summary and "\n" in desc: desc = " ".join(desc.splitlines()[:2]) plugins.log.info(prefix + repr(test) + " " + desc)