def run(self, files):
"""Process all files"""
self.files = files
t1 = time.clock()
for fn in files:
s, e = g.readFileIntoString(fn)
if s:
self.tot_s += len(s)
g.trace("%8s %s" % ("{:,}".format(len(s)), g.shortFileName(fn)))
# Print len(s), with commas.
# Fast, accurate:
# 1.9 sec for parsing.
# 2.5 sec for Null AstFullTraverer traversal.
# 2.7 sec to generate all strings.
# 3.8 sec to generate all reports.
s1 = g.toEncodedString(s)
self.tot_lines += len(g.splitLines(s))
# Adds less than 0.1 sec.
node = ast.parse(s1, filename="before", mode="exec")
ShowDataTraverser(self, fn).visit(node)
# elif 0: # Too slow, too clumsy: 3.3 sec for tokenizing
# readlines = g.ReadLinesClass(s).next
# for token5tuple in tokenize.generate_tokens(readlines):
# pass
# else: # Inaccurate. 2.2 sec to generate all reports.
# self.scan(fn, s)
else:
g.trace("skipped", g.shortFileName(fn))
t2 = time.clock()
# Get the time exlusive of print time.
self.show_results()
g.trace("done: %4.1f sec." % (t2 - t1))
def test_syntax_of_setup_py(self):
fn = g.os_path_finalize_join(g.app.loadDir, '..', '..', 'setup.py')
# Only run this test if setup.py exists: it may not in the actual distribution.
if not g.os_path_exists(fn):
self.skipTest('setup.py not found') # pragma: no cover
s, e = g.readFileIntoString(fn)
assert self.check_syntax(fn, s)
def readFile (self,fileName,root):
trace = False and not g.unitTesting
verbose = True
c = self.c
if not g.enableDB:
if trace: g.trace('g.enableDB is False')
return '',False,None
s,e = g.readFileIntoString(fileName,raw=True,silent=True)
if s is None:
if trace: g.trace('empty file contents',fileName)
return s,False,None
assert not g.isUnicode(s)
if trace and verbose:
for i,line in enumerate(g.splitLines(s)):
print('%3d %s' % (i,repr(line)))
# There will be a bug if s is not already an encoded string.
key = self.fileKey(root.h,s,requireEncodedString=True)
ok = self.db and key in self.db
if trace: g.trace('in cache',ok,fileName,key)
if ok:
# Delete the previous tree, regardless of the @<file> type.
while root.hasChildren():
root.firstChild().doDelete()
# Recreate the file from the cache.
aList = self.db.get(key)
self.createOutlineFromCacheList(root.v,aList,fileName=fileName)
return s,ok,key
def run(self, files):
'''Process all files'''
self.files = files
t1 = time.clock()
for fn in files:
s, e = g.readFileIntoString(fn)
if s:
self.tot_s += len(s)
g.trace('%8s %s' %
("{:,}".format(len(s)), g.shortFileName(fn)))
# Print len(s), with commas.
# Fast, accurate:
# 1.9 sec for parsing.
# 2.5 sec for Null AstFullTraverer traversal.
# 2.7 sec to generate all strings.
# 3.8 sec to generate all reports.
s1 = g.toEncodedString(s)
self.tot_lines += len(g.splitLines(s))
# Adds less than 0.1 sec.
node = ast.parse(s1, filename='before', mode='exec')
ShowDataTraverser(self, fn).visit(node)
# elif 0: # Too slow, too clumsy: 3.3 sec for tokenizing
# readlines = g.ReadLinesClass(s).next
# for token5tuple in tokenize.generate_tokens(readlines):
# pass
# else: # Inaccurate. 2.2 sec to generate all reports.
# self.scan(fn, s)
else:
g.trace('skipped', g.shortFileName(fn))
t2 = time.clock()
# Get the time exlusive of print time.
self.show_results()
g.trace('done: %4.1f sec.' % (t2 - t1))
def readFileIntoNode(self, event=None):
'''Read a file into a single node.'''
c = self
undoType = 'Read File Into Node'
c.endEditing()
filetypes = [
("All files", "*"),
("Python files", "*.py"),
("Leo files", "*.leo"),
]
fileName = g.app.gui.runOpenFileDialog(c,
title="Read File Into Node",
filetypes=filetypes,
defaultextension=None)
if not fileName: return
s, e = g.readFileIntoString(fileName)
if s is None:
return
g.chdir(fileName)
s = '@nocolor\n' + s
w = c.frame.body.wrapper
p = c.insertHeadline(op_name=undoType)
p.setHeadString('@read-file-into-node ' + fileName)
p.setBodyString(s)
w.setAllText(s)
c.redraw(p)
def diff(self, event=None):
'''Creates a node and puts the diff between 2 files into it.'''
c = self.c
fn = self.getReadableTextFile()
if not fn: return
fn2 = self.getReadableTextFile()
if not fn2: return
s1, e = g.readFileIntoString(fn)
if s1 is None: return
s2, e = g.readFileIntoString(fn2)
if s2 is None: return
lines1, lines2 = g.splitLines(s1), g.splitLines(s2)
aList = difflib.ndiff(lines1, lines2)
p = c.p.insertAfter()
p.h = 'diff'
p.b = ''.join(aList)
c.redraw()
def diff(self, event=None):
'''Creates a node and puts the diff between 2 files into it.'''
c = self.c
fn = self.getReadableTextFile()
if not fn: return
fn2 = self.getReadableTextFile()
if not fn2: return
s1, e = g.readFileIntoString(fn)
if s1 is None: return
s2, e = g.readFileIntoString(fn2)
if s2 is None: return
lines1, lines2 = g.splitLines(s1), g.splitLines(s2)
aList = difflib.ndiff(lines1, lines2)
p = c.p.insertAfter()
p.h = 'diff'
p.b = ''.join(aList)
c.redraw()
def diff(self, event):
'''Creates a node and puts the diff between 2 files into it.'''
w = self.editWidget(event)
if not w: return
fn = self.getReadableTextFile()
if not fn: return
fn2 = self.getReadableTextFile()
if not fn2: return
s1, e = g.readFileIntoString(fn)
if s1 is None: return
s2, e = g.readFileIntoString(fn2)
if s2 is None: return
# self.switchToBuffer(event,"*diff* of ( %s , %s )" % (name,name2))
data = difflib.ndiff(s1, s2)
idata = []
for z in data:
idata.append(z)
w.delete(0, 'end')
w.insert(0, ''.join(idata))
def import_txt_file(c, fn):
"""Import the .txt file into a new node."""
u = c.undoer
g.setGlobalOpenDir(fn)
undoData = u.beforeInsertNode(c.p)
last = c.lastTopLevel()
p = last.insertAfter()
p.h = f"@edit {fn}"
s, e = g.readFileIntoString(fn, kind='@edit')
p.b = s
u.afterInsertNode(p, 'Import', undoData)
c.setChanged()
c.redraw(p)
def update_open_with_node(self, ef):
"""Update the body text of ef.p to the contents of ef.path."""
assert isinstance(ef, ExternalFile), ef
c, p = ef.c, ef.p.copy()
g.blue(f"updated {p.h}")
s, e = g.readFileIntoString(ef.path)
p.b = s
if c.config.getBool('open-with-goto-node-on-update'):
c.selectPosition(p)
if c.config.getBool('open-with-save-on-update'):
c.save()
else:
p.setDirty()
c.setChanged()
def replaceFileWithString(self, fn, s):
'''Replace the file with s if s is different from theFile's contents.
Return True if theFile was changed.
'''
trace = False and not g.unitTesting
verbose = False
c = self.c
x = self
exists = g.os_path_exists(fn)
if exists:
# Read the file. Return if it is the same.
s2, e = g.readFileIntoString(fn)
if s2 is None:
return False
if s == s2:
report = c.config.getBool('report_unchanged_files',
default=True)
if report and not g.unitTesting:
g.es('unchanged:', fn)
return False
# Issue warning if directory does not exist.
theDir = g.os_path_dirname(fn)
if theDir and not g.os_path_exists(theDir):
if not g.unitTesting:
x.error('not written: %s directory not found' % fn)
return False
# Replace the file.
try:
f = open(fn, 'wb')
# 2011/09/09: Use self.encoding.
f.write(g.toEncodedString(s, encoding=self.encoding))
if trace:
g.trace('encoding', self.encoding)
if verbose:
g.trace(
'fn', fn,
'\nlines...\n%s' % (g.listToString(g.splitLines(s))),
'\ncallers', g.callers(4))
f.close()
if not g.unitTesting:
# g.trace('created:',fn,g.callers())
if exists: g.es('wrote:', fn)
else: g.es('created:', fn)
return True
except IOError:
x.error('unexpected exception writing file: %s' % (fn))
g.es_exception()
return False
def update_open_with_node(self, ef):
'''Update the body text of ef.p to the contents of ef.path.'''
trace = False and not g.unitTesting
assert isinstance(ef, ExternalFile), ef
if trace: g.trace(repr(ef))
c, p = ef.c, ef.p.copy()
# Ask the user how to resolve the conflict.
if self.ask(c, ef.path, p=p):
g.blue('updated %s' % p.h)
s, e = g.readFileIntoString(ef.path)
p.b = s
if c.config.getBool('open_with_goto_node_on_update'):
c.selectPosition(p)
if c.config.getBool('open_with_save_on_update'):
c.save()
def insertFile(self, event):
'''Prompt for the name of a file and put the selected text into it.'''
w = self.editWidget(event)
if not w:
return
fn = self.getReadableTextFile()
if not fn:
return
s, e = g.readFileIntoString(fn)
if s:
self.beginCommand(w, undoType='insert-file')
i = w.getInsertPoint()
w.insert(i, s)
w.seeInsertPoint()
self.endCommand(changed=True, setLabel=True)
def update_open_with_node(self, ef):
'''Update the body text of ef.p to the contents of ef.path.'''
trace = False and not g.unitTesting
assert isinstance(ef, ExternalFile), ef
if trace: g.trace(repr(ef))
c, p = ef.c, ef.p.copy()
# Ask the user how to resolve the conflict.
if self.ask(c, ef.path, p=p):
g.blue('updated %s' % p.h)
s, e = g.readFileIntoString(ef.path)
p.b = s
if c.config.getBool('open_with_goto_node_on_update'):
c.selectPosition(p)
if c.config.getBool('open_with_save_on_update'):
c.save()
def insertFile(self, event):
'''Prompt for the name of a file and put the selected text into it.'''
w = self.editWidget(event)
if not w:
return
fn = self.getReadableTextFile()
if not fn:
return
s, e = g.readFileIntoString(fn)
if s:
self.beginCommand(w, undoType='insert-file')
i = w.getInsertPoint()
w.insert(i, s)
w.seeInsertPoint()
self.endCommand(changed=True, setLabel=True)
def update_open_with_node(self, ef):
'''Update the body text of ef.p to the contents of ef.path.'''
assert isinstance(ef, ExternalFile), ef
c, p = ef.c, ef.p.copy()
# Ask the user how to resolve the conflict.
if self.ask(c, ef.path, p=p):
g.blue(f"updated {p.h}")
s, e = g.readFileIntoString(ef.path)
p.b = s
if c.config.getBool('open-with-goto-node-on-update'):
c.selectPosition(p)
if c.config.getBool('open-with-save-on-update'):
c.save()
else:
p.setDirty()
c.setChanged()
def update_open_with_node(self, ef):
'''Update the body text of ef.p to the contents of ef.path.'''
assert isinstance(ef, ExternalFile), ef
c, p = ef.c, ef.p.copy()
# Ask the user how to resolve the conflict.
if self.ask(c, ef.path, p=p):
g.blue('updated %s' % p.h)
s, e = g.readFileIntoString(ef.path)
p.b = s
if c.config.getBool('open-with-goto-node-on-update'):
c.selectPosition(p)
if c.config.getBool('open-with-save-on-update'):
c.save()
else:
p.setDirty()
c.setChanged(True)
def replaceFileWithString (self,fn,s):
'''Replace the file with s if s is different from theFile's contents.
Return True if theFile was changed.
'''
trace = False and not g.unitTesting ; verbose = False
x = self
exists = g.os_path_exists(fn)
if exists:
# Read the file. Return if it is the same.
s2,e = g.readFileIntoString(fn)
if s2 is None:
return False
if s == s2:
if not g.unitTesting: g.es('unchanged:',fn)
return False
# Issue warning if directory does not exist.
theDir = g.os_path_dirname(fn)
if theDir and not g.os_path_exists(theDir):
if not g.unitTesting:
x.error('not written: %s directory not found' % fn)
return False
# Replace the file.
try:
f = open(fn,'wb')
# 2011/09/09: Use self.encoding.
f.write(g.toEncodedString(s,encoding=self.encoding))
if trace:
g.trace('encoding',self.encoding)
if verbose: g.trace('fn',fn,
'\nlines...\n%s' %(g.listToString(g.splitLines(s))),
'\ncallers',g.callers(4))
f.close()
if not g.unitTesting:
# g.trace('created:',fn,g.callers())
if exists: g.es('wrote:',fn)
else: g.es('created:',fn)
return True
except IOError:
x.error('unexpected exception writing file: %s' % (fn))
g.es_exception()
return False
def test_syntax_of_all_files(self):
skip_tuples = (
('extensions', 'asciidoc.py'),
('test', 'scriptFile.py'),
)
join = g.os_path_finalize_join
skip_list = [join(g.app.loadDir, '..', a, b) for a, b in skip_tuples]
n = 0
for theDir in ('core', 'external', 'extensions', 'modes', 'plugins',
'scripts', 'test'):
path = g.os_path_finalize_join(g.app.loadDir, '..', theDir)
self.assertTrue(g.os_path_exists(path), msg=path)
aList = glob.glob(g.os_path_join(path, '*.py'))
if g.isWindows:
aList = [z.replace('\\', '/') for z in aList]
for z in aList:
if z not in skip_list:
n += 1
fn = g.shortFileName(z)
s, e = g.readFileIntoString(z)
self.assertTrue(self.check_syntax(fn, s), msg=fn)
def replaceFileWithString(self, encoding, fileName, s):
'''
Replace the file with s if s is different from theFile's contents.
Return True if theFile was changed.
'''
x, c = self, self.c
exists = g.os_path_exists(fileName)
if exists:
# Read the file. Return if it is the same.
s2, e = g.readFileIntoString(fileName)
if s2 is None:
return False
if s == s2:
report = c.config.getBool('report-unchanged-files',
default=True)
if report and not g.unitTesting:
g.es('unchanged:', fileName)
return False
# Issue warning if directory does not exist.
theDir = g.os_path_dirname(fileName)
if theDir and not g.os_path_exists(theDir):
if not g.unitTesting:
x.error('not written: %s directory not found' % fileName)
return False
# Replace the file.
try:
with open(fileName, 'wb') as f:
# Fix bug 1243847: unicode error when saving @shadow nodes.
f.write(g.toEncodedString(s, encoding=encoding))
c.setFileTimeStamp(fileName)
# Fix #1053. This is an *ancient* bug.
if not g.unitTesting:
kind = 'wrote' if exists else 'created'
g.es('%-6s: %s' % (kind, fileName))
return True
except IOError:
x.error('unexpected exception writing file: %s' % (fileName))
g.es_exception()
return False
def beautify(options, path):
'''Beautify the file with the given path.'''
fn = g.shortFileName(path)
s, e = g.readFileIntoString(path)
if not s:
return
print('beautifying %s' % fn)
s1 = g.toEncodedString(s)
node1 = ast.parse(s1, filename='before', mode='exec')
readlines = g.ReadLinesClass(s).next
tokens = list(tokenize.generate_tokens(readlines))
beautifier = PythonTokenBeautifier(c=None)
beautifier.delete_blank_lines = not options.keep
s2 = beautifier.run(tokens)
s2_e = g.toEncodedString(s2)
node2 = ast.parse(s2_e, filename='before', mode='exec')
if compare_ast(node1, node2):
f = open(path, 'wb')
f.write(s2_e)
f.close()
else:
print('failed to beautify %s' % fn)
def readFileIntoNode(self, event=None):
'''Read a file into a single node.'''
c = self
undoType = 'Read File Into Node'
c.endEditing()
filetypes = [("All files", "*"), ("Python files", "*.py"), ("Leo files", "*.leo"),]
fileName = g.app.gui.runOpenFileDialog(c,
title="Read File Into Node",
filetypes=filetypes,
defaultextension=None)
if not fileName: return
s, e = g.readFileIntoString(fileName)
if s is None:
return
g.chdir(fileName)
s = '@nocolor\n' + s
w = c.frame.body.wrapper
p = c.insertHeadline(op_name=undoType)
p.setHeadString('@read-file-into-node ' + fileName)
p.setBodyString(s)
w.setAllText(s)
c.redraw(p)
def beautify(options, path):
'''Beautify the file with the given path.'''
fn = g.shortFileName(path)
s, e = g.readFileIntoString(path)
if not s:
return
print('beautifying %s' % fn)
s1 = g.toEncodedString(s)
node1 = ast.parse(s1, filename='before', mode='exec')
readlines = g.ReadLinesClass(s).next
tokens = list(tokenize.generate_tokens(readlines))
beautifier = PythonTokenBeautifier(c=None)
beautifier.delete_blank_lines = not options.keep
s2 = beautifier.run(tokens)
s2_e = g.toEncodedString(s2)
node2 = ast.parse(s2_e, filename='before', mode='exec')
if compare_ast(node1, node2):
f = open(path, 'wb')
f.write(s2_e)
f.close()
else:
print('failed to beautify %s' % fn)
def replaceFileWithString(self, encoding, fileName, s):
'''
Replace the file with s if s is different from theFile's contents.
Return True if theFile was changed.
'''
x, c = self, self.c
exists = g.os_path_exists(fileName)
if exists:
# Read the file. Return if it is the same.
s2, e = g.readFileIntoString(fileName)
if s2 is None:
return False
if s == s2:
report = c.config.getBool('report-unchanged-files', default=True)
if report and not g.unitTesting:
g.es('unchanged:', fileName)
return False
# Issue warning if directory does not exist.
theDir = g.os_path_dirname(fileName)
if theDir and not g.os_path_exists(theDir):
if not g.unitTesting:
x.error('not written: %s directory not found' % fileName)
return False
# Replace the file.
try:
with open(fileName, 'wb') as f:
# Fix bug 1243847: unicode error when saving @shadow nodes.
f.write(g.toEncodedString(s, encoding=encoding))
c.setFileTimeStamp(fileName)
# Fix #1053. This is an *ancient* bug.
if not g.unitTesting:
kind = 'wrote' if exists else 'created'
g.es('%-6s: %s' % (kind, fileName))
return True
except IOError:
x.error('unexpected exception writing file: %s' % (fileName))
g.es_exception()
return False
