def processResults(self, wordSet, results, attributeName): """ Process the results coming from a spell check operation """ matchGroups = [] for words in wordSet: xx = self.word_re.finditer(words[0]) for w in xx: if w: matchGroups.append((w, words[1])) # check for dups last = None lastX = None for words in wordSet: if words[1] is None: last = None continue xx = self.dup_re.finditer(words[0]) for w in xx: g = w.group(0).strip().lower() if last: # print("compare '{0}' and '{1}'".format(last, g)) if last == g: if self.interactive: self.Interact(words[1], w, -1, wordSet, words) else: if attributeName: log.error("Duplicate word found '{0}' in attribute '{1}'". format(lastX, attributeName), where=words[1]) else: log.error("Duplicate word found '{0}'".format(lastX), where=words[1]) last = g lastX = w.group(0).strip()
def checkWord(self, wordToCheck): # Write word to check to the speller newLine = u'^ ' + wordToCheck + u'\n' if self.iso8859: log.note(u"Pre Encode = " + newLine) newLine = newLine.encode('iso-8859-1', 'replaceWithSpace') newLine = newLine.decode('iso-8859-1') else: newLine = newLine # .encode('utf-8') log.note(newLine) self.stdin.write(newLine) result = [] # Read all of the results while True: line = self.stdout.readline() if six.PY2: if self.iso8859: # log.note(" ".join("{:02x}".format(c) for c in line)) line = line.decode('iso-8859-1') else: line = line.decode('utf-8') line = line.strip() log.note('spell out line = ' + line) # Empty lines mean that we are done if len(line) == 0: break # '*' means ???? if line[0] == '*': continue m = self.aspell_re.match(line) if not m: log.error( "Internal error trying to match the line '{0}'".format( line)) continue if line[0] == '#': offset = int(m.group(2)) options = None elif line[0] == '&': offset = int(m.group(4)) options = m.group(5) else: log.error( "internal error - aspell says line is '{0}'".format(line)) continue tuple = (line[0], offset, None, m.group(1), options, 0) result.append(tuple) return result
def validate(self, tree): stdin = io.StringIO() xtract = SourceExtracter(tree, "abnf") if not xtract.ExtractToFile(stdin): log.info("No ABNF to check") return False cmds = [self.abnfProgram, "-q"] if self.dictionaries: for dict in self.dictionaries: if not os.path.exists(dict): raise RfcLintError( "Additional ABNF rule file '{0}' does not exist". format(dict)) cmds.append("-i") cmds.append(dict) p = subprocess.Popen(cmds, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate(stdin.getvalue().encode('utf-8')) errs = stderr.decode('utf-8').splitlines() noError = True noWarn = True for err in errs: if err.strip() == "": continue m = re.match(r"(.+)\((\d+):(\d+)\): error: (.*)", err) if m: line = int(m.group(2)) filename = m.group(1) if filename == 'stdin': runningLine = 0 for xxx in xtract.lineOffsets: if line < runningLine + xxx[2]: log.error(m.group(4), file=xxx[0], line=xxx[1] + line - runningLine) noError = False break runningLine += xxx[2] - 1 else: log.error(m.group(4), file=filename, line=line) noError = False else: log.info(err) noWarn = False if noError and noWarn: log.info("ABNF checked with no warnings or errors") elif noError: log.info("ABNF checked with no errors") return True
def insertBefore(self, siblingNode, newNode): i = 0 for child in siblingNode.parent.children: if siblingNode.isMatchNode(child): break i += 1 if i == len(siblingNode.parent.children): # assert false log.error("ICE: insertBefore failed to find node.") siblingNode.parent.children.append(newNode) else: siblingNode.parent.children.insert(i, newNode) newNode.parent = siblingNode.parent
def insertAfter(self, siblingNode, newNode): """ Insert newNode after siblingNode """ i = 0 for child in siblingNode.parent.children: if siblingNode.isMatchNode(child): break i += 1 if i == len(siblingNode.parent.children): log.error("ICE: insertAfter failed to find node.") siblingNode.parent.children.append(newNode) else: siblingNode.parent.children.insert(i + 1, newNode) newNode.parent = siblingNode.parent
def checkTree(self, tree): if tree.tag == 'bcp14': if not tree.text: inner = lxml.etree.tostring(tree, with_tail=False) if not isinstance(inner, type('')): inner = inner.decode('utf8') log.error( u"text '{0}' in bcp14 tag is not bcp14 language".format( inner), where=tree) elif not self.bcp14_re.match(tree.text): log.error( u"text '{0}' in bcp14 tag is not bcp14 language".format( tree.text), where=tree) elif tree.text: xx = self.bcp14_re.search(tree.text) if xx: if self.rewrite: xx = self.bcp14_re.search(tree.text) bcpNode = lxml.etree.SubElement(tree, "bcp14") bcpNode.text = xx.group(0) bcpNode.tail = tree.text[xx.end(0):] tree.text = tree.text[:xx.start(0)] tree.insert(0, bcpNode) else: xx = self.bcp14_re.finditer(tree.text) for x in xx: log.error( u"bcp14 text '{0}' found without bcp14 tag around it" .format(x.group(0)), where=tree) if tree.tail: xx = self.bcp14_re.search(tree.tail) if xx: if self.rewrite: parent = tree.getparent() bcpNode = lxml.etree.SubElement(parent, "bcp14") bcpNode.text = xx.group(0) bcpNode.tail = tree.tail[xx.end(0):] tree.tail = tree.tail[:xx.start(0)] parent.insert(parent.index(tree) + 1, bcpNode) self.checkTree(bcpNode) else: xx = self.bcp14_re.finditer(tree.text) for x in xx: log.error( u"bcp14 text '{0}' found without bcp14 tag around it" .format(x.group(0)), where=tree) for node in tree.iterchildren(): self.checkTree(node)
def initscr(self): self.A_REVERSE = 1 self.A_NORMAL = 0 if self.interactive and not self.no_curses: if haveCurses: try: self.curses = curses.initscr() curses.start_color() curses.noecho() curses.cbreak() self.spaceline = " " * curses.COLS self.A_REVERSE = curses.A_REVERSE self.A_NORMAL = curses.A_NORMAL except curses.error as e: if self.curses: self.endwin() self.curses = None log.error("Problem loading curses - " + e) else: log.warn("Unable to load CURSES for python")
def modify_style(node): """ For style properties, we want to pull it apart and then make individual attributes """ log.note("modify_style check '{0}' in '{1}'".format( node.attrib['style'], node.tag)) style_props = node.attrib['style'].rstrip(';').split(';') props_to_check = wp.style_properties for prop in style_props: # print("prop = %s" % prop) v = prop.split(':') if len(v) != 2: log.error( "Malformed field '{0}' in style attribute found. Field removed." .format(v), where=node) continue p = v[0].strip() v = v[1].strip() # May have leading blank log.note(" modify_style - p={0} v={1}".format(p, v)) # we will deal with the change of values later when the attribute list is processed. if p in props_to_check: log.error("Style property '{0}' promoted to attribute".format(p), where=node) node.attrib[p] = v else: log.error("Style property '{0}' removed".format(p), where=node) del node.attrib['style']
def DistanceTest(leftFile, rightFile, options): try: left = XmlRfcParser(leftFile, quiet=True).parse() left = BuildDiffTree(left.tree, options) left = AddParagraphs(left) right = XmlRfcParser(rightFile, quiet=True).parse() right = BuildDiffTree(right.tree, options) right = AddParagraphs(right) editSet = distance(left, right, DiffRoot.get_children, DiffRoot.InsertCost, DiffRoot.DeleteCost, DiffRoot.UpdateCost) c = left.applyEdits(editSet) if c > 0: log.error("Fail applying edits for '{0}' and '{1}', # edits = {2}". format(leftFile, rightFile, c)) except Exception as e: log.exception( "Fail applying edits for '{0}' and '{1}'".format( leftFile, rightFile), e)
def Interact(self, element, match, srcLine, wordSet, words, spellInfo): # # At the request of the RFC editors we use the ispell keyboard mappings # # R - replace the misspelled word completely # Space - Accept the word this time only. # A - Accept word for this session. # I - Accept the word, insert private dictionary as is # U - Accept the word, insert private dictionary as lower-case # 0-n - Replace w/ one of the suggested words # L - Look up words in the system dictionary - NOT IMPLEMENTED # X - skip to end of the file # Q - quit and don't save the file # ? - Print help # if self.curses: self.curses.erase() self.curses.move(0, 0) fileName = element.base if fileName.startswith("file:///"): fileName = os.path.relpath(fileName[8:]) elif fileName[0:6] == 'file:/': fileName = os.path.relpath(fileName[6:]) elif fileName[0:7] == 'http://' or fileName[0:8] == 'https://': pass else: fileName = os.path.relpath(fileName) y = 0 self.x = 0 self.y = 0 if isinstance(words[2], str): str1 = u"{1}:{2} Misspelled word '{0}' found in attribute '{3}'". \ format(match.group(0), fileName, element.sourceline, words[2]) else: str1 = u"{1}:{2} Misspelled word found '{0}'". \ format(match.group(0), fileName, element.sourceline) if self.curses: self.curses.addstr(curses.LINES - 15, 0, self.spaceline, curses.A_REVERSE) self.curses.addstr(curses.LINES - 14, 0, str1.encode('ascii', 'replaceWithONE')) self.curses.addstr(curses.LINES - 13, 0, self.spaceline, curses.A_REVERSE) else: log.write("") log.error(str1) self.writeStringInit() for line in wordSet: if isinstance(line[2], str): text = line[1].attrib[line[2]] elif line[1] is None: text = line[0] elif line[2]: text = line[1].text else: text = line[1].tail if words == line: if self.lastElement != line[1] or self.textLocation != line[2]: self.offset = 0 self.lastElement = line[1] self.textLocation = line[2] self.writeString(text[:match.start() + self.offset], partialString=True) self.writeString( text[match.start() + self.offset:match.end() + self.offset], self.A_REVERSE, True) self.writeString(text[match.end() + self.offset:]) else: self.writeString(text) y += 1 self.writeStringEnd() if spellInfo[4] is None: suggest = [] else: suggest = spellInfo[4].split(',') # list = "" # for i in range(min(10, len(suggest))): # list += "{0}) {1} ".format(chr(i + 0x31), suggest[i]) for i in range(min(10, len(suggest))): str1 = u"{0}) {1}".format((i + 1) % 10, suggest[i].strip()) if self.curses: self.curses.addstr( int(i / 2) + curses.LINES - 12, int(i % 2) * 40, str1) else: log.write_on_line(str1 + " ") if self.curses: self.curses.addstr(curses.LINES - 6, 0, " ) Ignore") self.curses.addstr(curses.LINES - 6, 40, "A) Accept Word Always") self.curses.addstr(curses.LINES - 5, 0, "I) Add to dictionary") self.curses.addstr(curses.LINES - 5, 40, "U) Add to dictionary lowercase") self.curses.addstr(curses.LINES - 4, 0, "D) Delete Word") self.curses.addstr(curses.LINES - 4, 40, "R) Replace Word") self.curses.addstr(curses.LINES - 3, 0, "Q) Quit") self.curses.addstr(curses.LINES - 3, 40, "X) Exit Spell Check") self.curses.addstr(curses.LINES - 2, 0, self.spaceline, curses.A_REVERSE) self.curses.addstr(curses.LINES - 1, 0, "?") self.curses.refresh() else: log.write("") replaceWord = None while (True): # ch = get_character() if self.curses: ch = chr(self.curses.getch()) else: ch = input("? ") ch = (ch + 'b')[0] if ch == ' ': return if ch == '?': if not self.curses: log.error( "HELP: ) Ignore, D) Delete Word, R) Replace Word, Q) Quit, X) Exit.", additional=0) elif ch == 'Q' or ch == 'q': if self.curses: self.curses.addstr(curses.LINES - 1, 0, "Are you sure you want to abort?") self.curses.refresh() ch = self.curses.getch() ch = chr(ch) else: ch = input("Are you sure you want to abort? ") ch = (ch + 'x')[0] if ch == 'Y' or ch == 'y': sys.exit(1) if self.curses: self.curses.addstr(curses.LINES - 1, 0, "?" + ' ' * 30) self.curses.refresh() elif ch == 'A' or ch == 'a': self.sendCommand("@" + match.group(0)) return elif ch == 'I' or ch == 'i': self.sendCommand("*" + match.group(0)) return elif ch == 'U' or ch == 'u': self.sendCommand("&" + match.group(0)) return elif ch == 'X' or ch == 'x': if self.curses: self.curses.addstr( curses.LINES - 1, 0, "Are you sure you want to exit spell checking?") self.curses.refresh() ch = self.curses.getch() ch = chr(ch) else: ch = input( "Are you sure you want to exit spell checking? ") ch = (ch + 'x')[0] if ch == 'Y' or ch == 'y': raise RfcLintError("Exit Requested") if self.curses: self.curses.addstr(curses.LINES - 1, 0, "?" + ' ' * 30) self.curses.refresh() elif ch == 'R' or ch == 'r': if self.curses: self.curses.addstr(curses.LINES - 1, 0, "Replace with: ") curses.nocbreak() curses.echo() self.curses.refresh() replaceWord = self.curses.getstr() curses.cbreak() curses.noecho() replaceWord = replaceWord.decode('utf-8') else: replaceWord = input("Replace with: ") elif 0x31 <= ord(ch) and ord(ch) <= 0x39: ch = ord(ch) - 0x31 replaceWord = suggest[ch] elif ch == '0': replaceWord = suggest[9] else: pass if replaceWord is not None: if isinstance(line[2], str): element.attrib[line[2]] = self.replaceText( element.attrib[line[2]], replaceWord, match, element) elif words[2]: element.text = self.replaceText(element.text, replaceWord, match, element) else: element.tail = self.replaceText(element.tail, replaceWord, match, element) return
def processResults(self, wordSet, results, attributeName): """ Process the results coming from a spell check operation """ matchGroups = [] allWords = [] if not self.interactive: for words in wordSet: if words[1] is None: continue newline = re.sub(r'\s*\n\s*', ' ', re.sub(r'\.\s*\n\s*', '. ', words[0])) xx = self.word_re.finditer(newline) for w in xx: if w: matchGroups.append((w, words[1], words[2], words[3])) allWords.append(w.group(1)) # if allWords[-1][-1] not in [' ', '-', "'"]: # allWords[-1] += ' ' if len(allWords) > 0: allWords[0] = allWords[0].lstrip() allWords[-1] = allWords[-1].rstrip() # do the spelling checking wordNo = -1 for words in wordSet: if words[1] is None: continue xx = self.spell_re.finditer(words[0]) for w in xx: wordNo += 1 sp = self.checkWord(w.group(0)) if len(sp) == 0: continue if self.interactive: self.Interact(words[1], w, -1, wordSet, words, sp[0]) else: if attributeName: log.error( "Misspelled word '{0}' in attribute '{1}'".format( w.group(0), attributeName), where=words[1]) else: log.error(u"Misspelled word was found '{0}'".format( w.group(0)), where=words[1]) if self.window > 0: if wordNo >= 0: ctx = "" if wordNo > 0: ctx = "".join( allWords[max(0, wordNo - self.window):wordNo]) ctx += self.color_start + allWords[ wordNo] + self.color_end if wordNo < len(allWords): ctx += "".join( allWords[wordNo + 1:min(wordNo + self.window + 1, len(allWords))]) log.error(ctx, additional=2) if self.suggest and sp[0][4]: suggest = " ".join(sp[0][4].split()[0:10]) log.error(suggest, additional=2)
def processLine(self, allWords): """ Process each individual set of words and return the errors found allWords is a tuple of (text string, tree element) returned is an array of tuples each tuple consisting of ( What the error is ('&' or '#'), What the word in error is, The set of alternative words (None for '#'), The offset in the string of the word, The word string, The tree node ) """ return [] result = [] setNo = 0 for wordSet in allWords: newLine = u'^ ' + wordSet[0] + u'\n' if self.iso8859: log.note(u"Pre Encode = " + newLine) newLine = newLine.encode('iso-8859-1', 'replaceWithSpace') newLine = newLine.decode('iso-8859-1') else: newLine = newLine # .encode('utf-8') log.note(newLine) self.stdin.write(newLine) while True: line = self.stdout.readline() if six.PY2: if self.iso8859: # log.note(" ".join("{:02x}".format(c) for c in line)) line = line.decode('iso-8859-1') else: line = line.decode('utf-8') line = line.strip() log.note('spell out line = ' + line) if len(line) == 0: break if line[0] == '*': continue m = self.aspell_re.match(line) if not m: log.error( "Internal error trying to match the line '{0}'".format( line)) continue if line[0] == '#': offset = int(m.group(2)) options = None elif line[0] == '&': offset = int(m.group(4)) options = m.group(5) else: log.error( "internal error - aspell says line is '{0}'".format( line)) continue tuple = (line[0], offset, wordSet[1], m.group(1), options, setNo) result.append(tuple) setNo += 1 return result
def validate(self, dtd_path=None, rng_path=None): """ Validate the document with its default dtd, or an optional one Return a success bool along with a list of any errors """ if rng_path: if os.path.exists(rng_path): try: rng = lxml.etree.parse(rng_path) rng = lxml.etree.RelaxNG(rng) except lxml.etree.XMLSyntaxError as e: log.error('Could not parse the rng file: ', rng_path + '\n ', e.message) return False, [] except lxml.etree.RelaxNGParseError as e: log.error('Could not parse the rng file: ', rng_path + '\n ', e.error_log.last_error.message) return False, [] else: # Invalid path given log.error("RNG file does not exist ", rng_path) return False, [] if rng.validate(self.tree): # The document was valid return True, [] else: if len(rng.error_log) == 0: return True, [] else: # The document was not valid return False, rng.error_log # Load dtd from alternate path, if it was specified if dtd_path: if os.path.exists(dtd_path): try: dtd = lxml.etree.DTD(dtd_path) except lxml.etree.DTDParseError as e: # The DTD itself has errors log.error('Could not parse the dtd file:', dtd_path + '\n ', e.message) return False, [] else: # Invalid path given log.error('DTD file does not exist:', dtd_path) return False, [] # Otherwise, use document's DTD declaration else: dtd = self.tree.docinfo.externalDTD if not dtd and self.default_dtd_path: # No explicit DTD filename OR declaration in document! log.warn('No DTD given, defaulting to', self.default_dtd_path) return self.validate(dtd_path=self.default_dtd_path) if not dtd or dtd.validate(self.getroot()): # The document was valid return True, [] else: if len(dtd.error_log) == 0: return True, [] else: # The document was not valid return False, dtd.error_log
def __init__(self, config): CursesCommon.__init__(self, config) program = config.get('spell', 'program') self.suggest = config.getBoolean('spell', 'suggest', True) self.window = config.getInt('spell', 'window', 15) coloring = config.get('spell', 'color') if coloring and coloring in SpellerColors: self.color_start = SpellerColors[coloring] self.color_end = colorama.Style.RESET_ALL if self.color_start == '': self.color_end = self.color_start elif os.name == 'nt': self.color_start = '' self.color_end = '' else: self.color_start = colorama.Style.BRIGHT self.color_end = colorama.Style.RESET_ALL if program: look_for = which(program) if not look_for and os.name == 'nt': look_for = which(program + '.exe') if not look_for: raise RfcLintError( "The program '{0}' does not exist or is not executable". format(program)) program = look_for else: if os.name == "nt": look_for = "aspell.exe" program = which(look_for) if not program: program = which( "c:\\Program Files (x86)\\Aspell\\bin\\aspell.exe") else: look_for = "aspell" program = which(look_for) if not program: raise RfcLintError( "The program '{0}' does not exist or is not executable". format(look_for)) spellBaseName = os.path.basename(program) spellBaseName = spellBaseName.replace('.exe', '') # I want to know what the program and version really are p = subprocess.Popen([program, "-v"], stdout=subprocess.PIPE) (versionOut, stderr) = p.communicate() """ if p.returncode != 0: raise RfcLintError("The program '{0}' executed with an error code {1}". format(program, p.returncode)) """ m = re.match( r".*International Ispell Version [\d.]+ \(but really (\w+) ([\d.]+).*", versionOut.decode('utf-8')) if m is None: raise RfcLintError( "Error starting the spelling program\n{0}".format(program)) if m.group(1).lower() != spellBaseName: raise RfcLintError( "Error: The wrong spelling program was started. Expected" "{0} and got {1}".format(spellBaseName, m.group(1))) codecs.register_error('replaceWithSpace', ReplaceWithSpace) self.iso8859 = False if spellBaseName == 'aspell': log.note("xx - " + m.group(2)) if m.group(2)[:3] == '0.5': # This version does not support utf-8 self.iso8859 = True log.note("Use iso8859") elif spellBaseName == 'hunspell': # minumum version of hunspell is 1.1.6, but that is probably not something # we would find in the wild anymore. We are therefore not going to check it. # However, right now the only version I have for Windows does not support utf-8 # so until I get a better version, force the use of iso8859 there. if os.name == 'nt': self.iso8859 = True log.note("Use iso8859") # now let's build the full command cmdLine = [program, '-a'] # always use pipe mode dicts = config.getList('spell', 'dictionaries') if dicts: dictList = '' for dict in dicts: if spellBaseName == 'hunspell': dict = dict + '.dic' if os.path.isabs(dict): dict2 = dict else: dict2 = os.path.join(os.getcwd(), dict) dict2 = os.path.normpath(dict2) if not os.path.exists(dict2): log.error( "Additional Dictionary '{0}' ignored because it was not found" .format(dict.replace('.dic', ''))) continue if spellBaseName == 'aspell': cmdLine.append("--add-extra-dicts") cmdLine.append(dict2) else: dictList = dictList + "," + dict2.replace('.dic', '') if spellBaseName == 'hunspell': cmdLine.append('-d') cmdLine.append("en_US" + dictList) dict = config.get('spell', 'personal') if dict: if os.path.isabs(dict): dict2 = dict else: dict2 = os.path.join(os.getcwd(), dict) dict2 = os.path.normpath(dict2) if not os.path.exists(dict2): log.error( "Personal Dictionary '{0}' ignored because it was not found" .format(dict)) else: cmdLine.append('-p') cmdLine.append(dict2) if self.iso8859: if spellBaseName == 'aspell': cmdLine.append('--encoding=iso8859-1') else: # Make sure if we have a better version of hunspell that it will do the right thing cmdLine.append('-i iso-8859-1') elif spellBaseName == 'hunspell': cmdLine.append('-i utf-8') log.note("spell command = '{0}'".format(" ".join(cmdLine))) self.p = subprocess.Popen(cmdLine, stdin=subprocess.PIPE, stdout=subprocess.PIPE) if six.PY2: if os.name == 'nt': self.stdin = codecs.getwriter('iso-8859-1')(self.p.stdin) self.stdout = self.p.stdout else: self.stdin = codecs.getwriter('utf8')(self.p.stdin) self.stdout = self.p.stdout # self.stdout = codecs.getreader('utf8')(self.p.stdout) else: if self.iso8859: self.stdin = io.TextIOWrapper(self.p.stdin, encoding='iso-8859-1', errors='replaceWithSpace', line_buffering=True) self.stdout = io.TextIOWrapper(self.p.stdout, encoding='iso-8859-1', errors='replaceWithSpace') else: self.stdin = io.TextIOWrapper(self.p.stdin, encoding='utf-8', line_buffering=True) self.stdout = io.TextIOWrapper(self.p.stdout, encoding='utf-8') # Check that we got a good return line = self.stdout.readline() log.note(line) if line.lower().find('aspell') < 0: raise RfcLintError( "The program '{0}' did not start successfully".format(program)) # [^\W\d_] self.word_re = re.compile(r'(\W*\w+\W*)', re.UNICODE | re.MULTILINE) self.word_re = re.compile(r'([\W\d_]*[^\W\d_]+[\W\d_]*)', re.UNICODE | re.MULTILINE) # self.word_re = re.compile(r'\w+', re.UNICODE | re.MULTILINE) # pattern to match output of aspell self.aspell_re = re.compile(r".\s(\S+)\s(\d+)\s*((\d+): (.+))?", re.UNICODE) self.spell_re = re.compile(r'\w[\w\'\u00B4\u2019]*\w', re.UNICODE) self.spell_re = re.compile( r'[^\W\d_]([^\W\d_]|[\'\u00B4\u2019])*[^\W\d_]', re.UNICODE) if config.options.output_filename is not None: self.ignoreWords = [] self.lastElement = None self.textLocation = True
def check(el, depth=0): """ Walk the current tree checking to see if all elements pass muster relative to RFC 7996 the RFC Tiny SVG document Return False if the element is to be removed from tree when writing it back out """ global errorCount log.note("%s tag = %s" % (' ' * (depth * indent), el.tag)) # Check that the namespace is one of the pre-approved ones # ElementTree prefixes elements with default namespace in braces element, ns = strip_prefix(el.tag, el) # name of element # namespace for elements must be either empty or svg if ns is not None and ns not in wp.svg_urls: log.warn("Element '{0}' in namespace '{1}' is not allowed".format( element, ns), where=el) return False # Remove this el # Is the element in the list of legal elements? log.note("%s element % s: %s" % (' ' * (depth * indent), element, el.attrib)) if element not in wp.elements: errorCount += 1 log.warn("Element '{0}' not allowed".format(element), where=el) return False # Remove this el elementAttributes = wp.elements[element] # Allowed attributes for element # do a re-write of style into individual elements if 'style' in el.attrib: modify_style(el) attribs_to_remove = [] # Can't remove them inside the iteration! for nsAttrib, val in el.attrib.items(): # validate that the namespace of the element is known and ok attr, ns = strip_prefix(nsAttrib, el) log.note("%s attr %s = %s (ns = %s)" % (' ' * (depth * indent), attr, val, ns)) if ns is not None and ns not in wp.svg_urls: if ns not in wp.xmlns_urls: log.warn( "Element '{0}' does not allow attributes with namespace '{1}'" .format(element, ns), where=el) attribs_to_remove.append(nsAttrib) continue # look to see if the attribute is either an attribute for a specific # element or is an attribute generically for all properties if (attr not in elementAttributes) and (attr not in wp.properties): errorCount += 1 log.warn("The element '{0}' does not allow the attribute '{1}'," " attribute to be removed.".format(element, attr), where=el) attribs_to_remove.append(nsAttrib) # Now check if the attribute is a generic property elif (attr in wp.properties): vals = wp.properties[attr] # log.note("vals = " + vals + "<<<<<") # Do method #1 of checking if the value is legal - not currently used. if vals and vals[0] == '[' and False: # ok, new_val = check_some_props(attr, val, depth) # if not ok: # el.attrib[attr] = new_val[1:] pass else: ok, new_val = value_ok(attr, val) if vals and not ok: errorCount += 1 if new_val is not None: el.attrib[attr] = new_val log.warn( u"The attribute '{1}' does not allow the value '{0}'," u" replaced with '{2}'".format(val, attr, new_val), where=el) else: attribs_to_remove.append(nsAttrib) log.warn( u"The attribute '{1}' does not allow the value '{0}'," u" attribute to be removed".format(val, attr), where=el) for attrib in attribs_to_remove: del el.attrib[attrib] # Need to have a viewBox on the root if (depth == 0): if el.get("viewBox"): pass else: log.warn( "The attribute viewBox is required on the root svg element", where=el) svgw = maybefloat(el.get('width')) svgh = maybefloat(el.get('height')) try: if svgw and svgh: newValue = '0 0 %s %s' % (svgw, svgh) log.warn("Trying to put in the attribute with value '{0}'". format(newValue), where=el) el.set('viewBox', newValue) except ValueError as e: log.error("Error when calculating SVG size: %s" % e, where=el) els_to_rm = [] # Can't remove them inside the iteration! if element in wp.element_children: allowed_children = wp.element_children[element] else: allowed_children = [] for child in el: log.note("%schild, tag = %s" % (' ' * (depth * indent), child.tag)) if not isinstance(child.tag, str): continue ch_tag, ns = strip_prefix(child.tag, el) if ns not in wp.svg_urls: log.warn( "The namespace {0} is not permitted for svg elements.".format( ns), where=child) els_to_rm.append(child) continue if ch_tag not in allowed_children: log.warn( "The element '{0}' is not allowed as a child of '{1}'".format( ch_tag, element), where=child) els_to_rm.append(child) elif not check(child, depth + 1): els_to_rm.append(child) if len(els_to_rm) != 0: for child in els_to_rm: el.remove(child) return True # OK
def main(): # Populate options formatter = optparse.IndentedHelpFormatter(max_help_position=40) optionparser = optparse.OptionParser(usage='rfclint SOURCE [OPTIONS] ' '...\nExample: rfclint ' 'draft.xml', formatter=formatter) parser_options = optparse.OptionGroup(optionparser, "Parser Options") parser_options.add_option('-C', '--clear-cache', action='store_true', dest='clear_cache', default=False, help='purge the cache and exit') parser_options.add_option('-c', '--cache', dest='cache', help='specify a primary cache directory to' ' write to; default: try [ %s ]' % ', '.join(CACHES)) parser_options.add_option( '-N', '--no-network', action='store_true', default=False, help='don\'t use the network to resolve references') parser_options.add_option('-n', '--no-rng', action='store_true', help='disable RNG validation step') parser_options.add_option('-r', '--rng', action='store_true', help='Specify an alternate RNG file') parser_options.add_option('-X', '--no-xinclude', action='store_true', dest='no_xinclude', help='don\'t resolve any xi:include elements') optionparser.add_option_group(parser_options) general_options = optparse.OptionGroup(optionparser, "General Options") general_options.add_option('-o', '--out', dest='output_filename', metavar='FILE', help='specify an explicit output filename') parser_options.add_option('--no-xml', dest='no_xml', action='store_true', help='Don\'t perform XML well-formness checking') parser_options.add_option('--bcp14', dest='bcp14', action='store_true', help='Perform bcp14 checking') optionparser.add_option_group(general_options) plain_options = optparse.OptionGroup(optionparser, 'Plain Options') plain_options.add_option('-q', '--quiet', action='store_true', help='dont print anything') plain_options.add_option('-v', '--verbose', action='store_true', help='print extra information') plain_options.add_option('-V', '--version', action='callback', callback=display_version, help='display the version number and exit') plain_options.add_option('--debug', action='store_true', help='Show debugging output') plain_options.add_option('--extract', dest='extract', help='Extract all items of the given type') plain_options.add_option('--no-svgcheck', action='store_true', dest='no_svgcheck', help='Don\'t run svgcheck') optionparser.add_option_group(plain_options) spell_options = optparse.OptionGroup(optionparser, 'Spell Options') spell_options.add_option('--no-spell', dest='no_spell', default=False, action='store_true', help='Don\'t run the spell checking') spell_options.add_option( '--dictionary', dest='dict_list', action='append', help='Use this addition dictionary when spell checking') spell_options.add_option( '--personal', dest='dict_personal', help='use this dictionary as the personal dictionary') spell_options.add_option( '--spell-window', dest='spell_window', action='store', type='int', help='Set the number of words to appear around spelling errors') spell_options.add_option('--no-dup-detection', dest='no_dups', action='store_true', help='Don\'t do duplication detection.') spell_options.add_option('--spell-program', dest='spell_program', metavar='NAME', help='Name of spelling program to use') spell_options.add_option('--no-suggest', dest='spell_suggest', action='store_false', help='Do not provide suggestions') spell_options.add_option('--suggest', dest='spell_suggest', action='store_true', help='provide suggestions (default)') spell_options.add_option('--color', dest='spell_color', action='callback', callback=check_color, type='string', help='color incorrect words in supplied context') spell_options.add_option( '--no-curses', dest='no_curses', action='store_true', help='disable curses when doing spell checking and dup detection') spell_options.add_option( '--skip-code', dest='skip_code', action='store_true', help='skip all code elements when doing spell and duplicate checking') spell_options.add_option( '--skip-artwork', dest='skip_artwork', action='store_true', help='skip all artwork elements when doing spell and ' 'duplicate checking') optionparser.add_option_group(spell_options) abnf_options = optparse.OptionGroup(optionparser, 'ABNF Options') abnf_options.add_option('--abnf-program', dest='abnf_program', metavar='NAME', help='Name of ABNF checker program to use') abnf_options.add_option('--no-abnf', dest='no_abnf', action='store_true', help='Don\'t perform ABNF checking') abnf_options.add_option('--abnf-add-rules', dest='abnf_add', help='ABNF file to append during evaluation.') config_options = optparse.OptionGroup(optionparser, 'Configuration Options') config_options.add_option( '--configfile', dest='config_file', metavar='NAME', help="Specify the name of the configuration file.") config_options.add_option('--save-config', dest='save_config', default=False, action='store_true', help='Save configuration back to file') # --- Parse and validate arguments --------------------------------- (options, args) = optionparser.parse_args() # --- Setup and parse the input file if options.cache: if not os.path.exists(options.cache): try: os.makedirs(options.cache) if options.verbose: log.write('Created cache directory at', options.cache) except OSError as e: print('Unable to make cache directory: %s ' % options.cache) print(e) sys.exit(1) else: if not os.access(options.cache, os.W_OK): print('Cache directory is not writable: %s' % options.cache) sys.exit(1) if options.clear_cache: clear_cache(options.cache) # --- Locate the configuration file if it exists and import it ---- config = ConfigFile(options) if options.save_config: config.save() sys.exit(0) # make things quiet if output goes to stdout if options.output_filename is None and not options.quiet and ( options.extract): options.quiet = True # --- Get the file to be processed -------------------------------- if len(args) < 1: optionparser.print_help() sys.exit(2) source = args[0] if not os.path.exists(source): sys.exit('No such file: ' + source) # Setup warnings module # rfclint.log.warn_error = options.warn_error and True or False log.quiet = options.quiet and True or False log.verbose = options.verbose # Parse the document into an xmlrfc tree instance log.note("Checking for well-formness of '{0}'".format(source)) parser = XmlRfcParser(source, verbose=options.verbose, preserve_all_white=True, quiet=True, cache_path=options.cache, no_network=options.no_network, no_xinclude=options.no_xinclude, templates_path=globals().get('_TEMPLATESPATH', None)) try: xmlrfc = parser.parse(remove_comments=False, strip_cdata=False) except XmlRfcError as e: log.exception('Unable to parse the XML document: ' + source, e) sys.exit(1) except lxml.etree.XMLSyntaxError as e: # Give the lxml.etree.XmlSyntaxError exception a line attribute which # matches lxml.etree._LogEntry, so we can use the same logging function log.error("Unable to parse the XML document: " + os.path.normpath(source)) log.exception_lines("dummy", e.error_log) sys.exit(1) log.note("Well-formness passes") # Validate the document unless disabled if not options.no_rng: log.note("Checking for schema validation...") if not options.rng: options.rng = parser.default_rng_path ok, errors = xmlrfc.validate(rng_path=options.rng) if not ok: log.error('Unable to validate the XML document: ' + os.path.normpath(source)) log.exception_lines("dummy", errors) sys.exit(1) log.info("Schema validation passes") else: log.note("Skipping schema validation") # Do Extracts if options.extract: codeItems = xmlrfc.tree.getroot().xpath( "//sourcecode[@type='{0}']".format(options.extract)) if len(codeItems) == 0: log.error("No sourcecode elements with type = '{0}' found.".format( options.extract)) exit(1) if options.output_filename: file = open(options.output_filename, 'w') else: file = sys.stdout needEOL = True for item in codeItems: if "name" in item.attrib: with open(item.attrib["name"], 'w') as f: f.write(item.text) if len(item.text) > 0 and item.text[-1] != '\n': f.write('\n') else: file.write(item.text) if len(item.text) > 0: needEOL = item.text[-1] != '\n' if needEOL: file.write('\n') if options.output_filename: file.close() exit(0) # Validate any embedded XML if not options.no_xml: codeItems = xmlrfc.tree.getroot().xpath("//sourcecode[@type='xml']") if len(codeItems) > 0: log.note("Validating XML fragments in sourcecode elements") # resolver without knowledge of rfc_number: caching_resolver = CachingResolver(no_network=True, verbose=options.verbose, quiet=options.quiet) for item in codeItems: parser = lxml.etree.XMLParser(dtd_validation=False, load_dtd=False, no_network=True, resolve_entities=False, recover=False) parser.resolvers.add(caching_resolver) try: text = re.sub(u'^\s+<\?xml ', '<?xml ', item.text) file = six.BytesIO(text.encode('utf-8')) lxml.etree.parse(file, parser) log.info( "XML fragment in source code found and is well defined.", where=item) except (lxml.etree.XMLSyntaxError) as e: log.warn(u'XML in sourcecode not well formed: ', e.msg, where=item) except Exception as e: log.exception(u'Error occured processing XML: ', e) else: log.info("No XML fragments in sourcecode elements found.") # Validate any embedded ABNF if not options.no_abnf: try: checker = AbnfChecker(config) checker.validate(xmlrfc.tree) except RfcLintError as e: log.error("Skipping ABNF checking because") log.error(e.message, additional=2) # Validate any SVG items if not options.no_svgcheck: checkTree(xmlrfc.tree) # do the Spelling checking if not options.no_spell: speller = None try: speller = Speller(config) if options.no_curses: speller.no_curses = True speller.initscr() speller.processTree(xmlrfc.tree.getroot()) speller.sendCommand("#") # save personal dictionary speller.endwin() except RfcLintError as e: log.error("Skipping spell checking because") log.error(e.message, additional=2) if speller: speller.endwin() except Exception: if speller: speller.endwin() raise # do the Duplicate checking if not options.no_dups: try: dups = Dups(config) if options.no_curses: dups.no_curses = True dups.initscr() dups.processTree(xmlrfc.tree.getroot()) dups.endwin() except RfcLintError as e: dups.endwin() log.error("Skipping duplicate checking because") log.error(e.message, additional=2) except Exception: dups.endwin() raise # do the 2119 Language tag checking if options.bcp14: try: lang2119 = Lang2119(config) if options.no_curses: lang2119.no_curses = True lang2119.initscr() lang2119.processTree(xmlrfc.tree.getroot()) lang2119.endwin() except RfcLintError as e: log.error("Skipping RFC 2119 language tag checking because") log.error(e.message, additoin=2) except Exception: lang2119.endwin() raise if options.output_filename is not None: if six.PY2: file = open(options.output_filename, 'w') else: file = open(options.output_filename, 'w', encoding='utf8') text = lxml.etree.tostring(xmlrfc.tree.getroot(), xml_declaration=True, encoding='utf-8', doctype=xmlrfc.tree.docinfo.doctype) if six.PY3: text = text.decode('utf8') file.write(text) if len(text) > 0 and text[-1] != '\n': file.write('\n')
def main(): # Populate the options formatter = optparse.IndentedHelpFormatter(max_help_position=40) optionparser = optparse.OptionParser(usage='svgcheck SOURCE [OPTIONS] ' '...\nExample: svgcheck draft.xml', formatter=formatter) parser_options = optparse.OptionGroup(optionparser, 'Parser Options') parser_options.add_option( '-N', '--no-network', action='store_true', default=False, help='don\'t use the network to resolve references') parser_options.add_option('-X', "--no-xinclude", action='store_true', default=False, help='don\'t resolve xi:include elements') parser_options.add_option('-C', '--clear-cache', action='store_true', dest='clear_cache', default=False, help='purge the cache and exit') parser_options.add_option( '-c', '--cache', dest='cache', help='specify a primary cache directory to write to;' 'default: try [ %s ]' % ', '.join(CACHES)) parser_options.add_option('-d', '--rng', dest='rng', help='specify an alternate RNG file') optionparser.add_option_group(parser_options) other_options = optparse.OptionGroup(optionparser, 'Other options') other_options.add_option('-q', '--quiet', action='store_true', help='don\'t print anything') other_options.add_option('-o', '--out', dest='output_filename', metavar='FILE', help='specify an explicit output filename') other_options.add_option('-v', '--verbose', action='store_true', help='print extra information') other_options.add_option('-V', '--version', action='callback', callback=display_version, help='display the version number and exit') optionparser.add_option_group(other_options) svg_options = optparse.OptionGroup(optionparser, 'SVG options') svg_options.add_option('-r', '--repair', action='store_true', default=False, help='Repair the SVG so it meets RFC 7966') svg_options.add_option( '-a', '--always-emit', action='store_true', default=False, help='Emit the SVG file even if does not need repairing. Implies -r') svg_options.add_option( '-g', '--grey-scale', action='store_true', help='Use grey scaling hieristic to determine what is white') svg_options.add_option( '--grey-level', default=381, help='Level to use for grey scaling, defaults to 381') optionparser.add_option_group(svg_options) # --- Parse and validate arguments -------------- (options, args) = optionparser.parse_args() # Setup warnings module # rfclint.log.warn_error = options.warn_error and True or False log.quiet = options.quiet and True or False log.verbose = options.verbose if options.cache: if not os.path.exists(options.cache): try: os.makedirs(options.cache) if options.verbose: log.write('Created cache directory at', options.cache) except OSError as e: print('Unable to make cache directory: %s ' % options.cache) print(e) sys.exit(1) else: if not os.access(options.cache, os.W_OK): print('Cache directory is not writable: %s' % options.cache) sys.exit(1) if options.clear_cache: clear_cache(options.cache) if options.grey_scale: wp.color_threshold = options.grey_level sourceText = None if len(args) < 1: sourceText = sys.stdin.read() source = os.getcwd() + "/stdin" else: source = args[0] if not os.path.exists(source): sys.exit('No such file: ' + source) # Setup warnings module # rfclint.log.warn_error = options.warn_error and True or False log.quiet = options.quiet and True or False log.verbose = options.verbose # Parse the document into an xmlrfc tree instance parser = XmlRfcParser(source, verbose=options.verbose, preserve_all_white=True, quiet=options.quiet, cache_path=options.cache, no_network=options.no_network, no_xinclude=options.no_xinclude) try: xmlrfc = parser.parse(remove_pis=True, remove_comments=False, strip_cdata=False, textIn=sourceText) except XmlRfcError as e: log.exception('Unable to parse the XML document: ' + source, e) sys.exit(1) except lxml.etree.XMLSyntaxError as e: # Give the lxml.etree.XmlSyntaxError exception a line attribute which # matches lxml.etree._LogEntry, so we can use the same logging function log.exception('Unable to parse the XML document: ' + source, e.error_log) sys.exit(1) # Check that ok = checkTree(xmlrfc.tree) if (not ok and options.repair) or options.always_emit: encodedBytes = lxml.etree.tostring(xmlrfc.tree.getroot(), xml_declaration=True, encoding='utf-8', pretty_print=True).decode('utf-8') if options.output_filename is None: file = sys.stdout else: file = open(options.output_filename, 'w') log.write_to(file, encodedBytes) if ok: log.info("File conforms to SVG requirements.") sys.exit(0) log.error("File does not conform to SVG requirements") sys.exit(1)
def Interact(self, element, match, srcLine, wordSet, words): if self.curses: self.curses.erase() self.curses.move(0, 0) fileName = element.base if fileName.startswith("file:///"): fileName = os.path.relpath(fileName[8:]) elif fileName[0:6] == 'file:/': fileName = os.path.relpath(fileName[6:]) elif fileName[0:7] == 'http://' or fileName[0:8] == 'https://': pass else: fileName = os.path.relpath(fileName) y = 0 self.x = 0 self.y = 0 if isinstance(words[2], str): str1 = u"{1}:{2} Duplicate word '{0}' found in attribute '{3}'". \ format(match.group(0), fileName, element.sourceline, words[2]) else: str1 = u"{1}:{2} Duplicate word found '{0}'". \ format(match.group(0), fileName, element.sourceline) if self.curses: self.curses.addstr(curses.LINES-14, 0, str1.encode('ascii', 'replaceWithONE')) else: log.write("") log.error(str1) self.writeStringInit() for line in wordSet: if isinstance(line[2], str): text = line[1].attrib[line[2]] elif line[1] is None: text = line[0] elif line[2]: text = line[1].text else: text = line[1].tail if words == line: if self.lastElement != line[1] or self.textLocation != line[2]: self.offset = 0 self.lastElement = line[1] self.textLocation = line[2] self.writeString(text[:match.start()+self.offset], partialString=True) self.writeString(text[match.start()+self.offset:match.end()+self.offset], self.A_REVERSE, True) self.writeString(text[match.end()+self.offset:]) else: self.writeString(text) y += 1 self.writeStringEnd() if self.curses: self.curses.addstr(curses.LINES-15, 0, self.spaceline, self.A_REVERSE) self.curses.addstr(curses.LINES-13, 0, self.spaceline, self.A_REVERSE) self.curses.addstr(curses.LINES-2, 0, self.spaceline, self.A_REVERSE) self.curses.addstr(curses.LINES-1, 0, "?") self.curses.addstr(curses.LINES-11, 0, " ) Ignore") self.curses.addstr(curses.LINES-10, 0, "D) Delete Word") self.curses.addstr(curses.LINES-9, 0, "R) Replace Word") self.curses.addstr(curses.LINES-8, 0, "Q) Quit") self.curses.addstr(curses.LINES-7, 0, "X) Exit Dup Check") self.curses.addstr(curses.LINES-1, 0, "?") self.curses.refresh() while (True): # ch = get_character() if self.curses: ch = chr(self.curses.getch()) else: ch = input("? ") ch = (ch + "b")[0] if ch == ' ': return if ch == '?': if not self.curses: log.error("HELP: ) Ignore, D) Delete Word, R) Replace Word, Q) Quit, X) Exit.", additional=0) elif ch == 'Q' or ch == 'q': if self.curses: self.curses.addstr(curses.LINES-1, 0, "Are you sure you want to abort?") self.curses.refresh() ch = self.curses.getch() ch = chr(ch) else: ch = input("Are you sure you want to abort? ") ch = (ch + 'x')[0] if ch == 'Y' or ch == 'y': self.endwin() sys.exit(1) if self.curses: self.curses.addstr(curses.LINES-1, 0, "?" + ' '*30) self.curses.refresh() elif ch == 'D' or ch == 'd': if isinstance(line[2], str): element.attrib[line[2]] = self.removeText(element.attrib[line[2]], match, element) elif words[2]: element.text = self.removeText(element.text, match, element) else: element.tail = self.removeText(element.tail, match, element) return elif ch == 'X' or ch == 'x': if self.curses: self.curses.addstr(curses.LINES-1, 0, "Are you sure you want to exit spell checking?") self.curses.refresh() ch = self.curses.getch() ch = chr(ch) else: ch = input("Are you sure you want to exit spell checking? ") ch = (ch + 'x')[0] if ch == 'Y' or ch == 'y': raise RfcLintError("Exit Requested") if self.curses: self.curses.addstr(curses.LINES-1, 0, "?" + ' '*30) self.curses.refresh() elif ch == 'R': if self.curses: self.curses.addstr(curses.LINES-1, 0, "Replace with: ") self.curses.refresh() ch = '' while True: ch2 = chr(self.curses.getch()) if ch2 == '\n': break ch += ch2 else: ch = input("Replace with: ") if isinstance(line[2], str): element.attrib[line[2]] = self.replaceText(element.attrib[line[2]], ch, match, element) elif words[2]: element.text = self.replaceText(element.text, ch, match, element) else: element.tail = self.replaceText(element.tail, ch, match, element) return else: pass