def build_imported_libraries_vocabulary(script_folder,
max_script_count=10000,
vocab_size=500,
min_count=2):
'''
Generates a dictionary of imported library calls to be used as the vocabulary in techniques that utilize bag of words.
Args:
script_folder (str): Folder location of corpus containing script files
max_script_count (int): the maximum number of code scripts to process in the script_folder
vocab_size (int): the maximum number of words to be used in the vocabulary (dimension of bag of words vector)
min_word_count (int): a word will be included in vocabulary if it appears at least min_count times in the corpus
Returns:
libraries_ordered_by_count (list): a list of size equal or less than max_vocab_size that contains the most frequent
normalized words in the corpus
'''
library_count = defaultdict(int)
counter = 0
# Retrieve files with script content and process with red baron to identify imported libraries
# Altair's JSON format uses the 'content' label for the script code
for py_file in sorted(os.listdir(script_folder)):
if counter >= max_script_count: break
fullpath = os.path.join(script_folder, py_file)
with open(fullpath, "r") as py_file_contents:
for line in py_file_contents:
counter += 1
parsed_json = json.loads(line)
try:
red = RedBaron(parsed_json['content'])
libraries = parse_import_statements(
red.find_all("ImportNode"))
libraries |= parse_fromimport_statements(
red.find_all("FromImportNode"))
for library in libraries:
library_count[library] += 1
except Exception as e:
#logger.info("%s error encountered in %s; skipping file" % (e.__class__.__name__, py_file))
continue
if counter >= max_script_count: break
# Determine descending order for library based on count
libraries_ordered_by_count = [
i[0] for i in sorted(
library_count.items(), key=lambda x: (x[1], x[0]), reverse=True)
if i[1] > min_count
]
# Trim the vocabulary to the requested vocab_size
if len(libraries_ordered_by_count) >= vocab_size:
libraries_ordered_by_count = libraries_ordered_by_count[:vocab_size]
else:
logger.warning("Only %d libraries were observed using max_script_count=%d, max_vocab_size=%d and min_word_count=%d" % \
(len(libraries_ordered_by_count), max_script_count, vocab_size, min_count))
return libraries_ordered_by_count
def _extract_libraries(self, document):
try:
red = RedBaron(document)
libraries = parse_import_statements(red.find_all("ImportNode"))
libraries |= parse_fromimport_statements(red.find_all("FromImportNode"))
# TODO: Skip document and remove from raw list if red baron fails parsing
except:
libraries = set()
logger.info("Red baron error;continuing")
return " ".join(libraries)
def functionalize(src):
red = RedBaron(src)
red.insert(0, 'import pync')
for func in red.find_all('def'):
func.decorators.append('@pync.curry')
for l in red.find_all('list') + red.find_all('list_comprehension'):
l.replace("pync.list(%s)" % l)
return red.dumps()
def load_real_code(code):
# dataset_folder = '/home/smartscript/smartscript_web/static/py_checker/misc/upload'
methods = []
codeNames = []
funcNames = []
# for code_file in glob.glob(os.path.join(dataset_folder, '*.py'), recursive=False):
# print(code_file)
# with open(code_file, 'r') as reader:
# code = reader.read()
red = RedBaron(code)
method_reds = red.find_all('def')
for method_red in method_reds:
funcName = str(method_red).split(':')[0][4:]
methods.append(method_red.dumps())
codeNames.append("uploaded source code")
funcNames.append(funcName)
# methods1 = []
# codeNames1 = []
# funcNames1 = []
# for m in methods:
# methods1.append(m)
# for c in codeNames:
# codeNames1.append(c)
# for f in funcNames:
# funcNames1.append(f)
# for m in methods:
# methods1.append(m)
# for c in codeNames:
# codeNames1.append(c)
# for f in funcNames:
# funcNames1.append(f)
return methods,codeNames,funcNames
def _cleanupPyLintComments(filename, abort):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
old_code = getFileContents(filename)
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def main() -> Optional[int]:
has_errors = False
for model_file_path in get_input_models_files():
with open(model_file_path) as f:
baron_tree = RedBaron(f.read())
for assignment in baron_tree.find_all('assignment'):
nodes_to_check = [
assignment.find('comment'),
]
if isinstance(assignment.next, CommentNode):
nodes_to_check.append(assignment.next)
assignment_str = assignment.dumps()
is_nullable_field = (
NULL_TRUE_KWARG_RE.search(assignment_str)
or 'NullBooleanField' in assignment_str
)
if is_nullable_field and not any(filter(has_valid_comment, nodes_to_check)):
has_errors = True
field_name = assignment.target
print( # noqa: T001
'{0}:{1} Field "{2}" needs a valid comment for its "null=True"'.format(
model_file_path, get_node_line(assignment), field_name,
),
)
if has_errors:
return 1
def compare(s1, s2, decay_factor = DEFAULT_DECAY_FACTOR):
red1 = RedBaron(s1)
red2 = RedBaron(s2)
result = []
for ast_f2 in red2.find_all('def'):
ast_f1 = red1.find('def', name = ast_f2.name)
if ast_f1 is not None:
additions, deletions = preprocess_files(ast_f1.dumps(),
ast_f2.dumps())
comments, exceptions = preprocess_comments(ast_f2, additions)
for a in additions:
for c in comments:
line, _ = c.left_bounds
distance = math.fabs(line - a)
score = int(c.score() - float(decay_factor) / (distance * distance))
c.setScore(score if score > 0 else 0)
for d in deletions:
for c in comments:
line, _ = c.left_bounds
line = line + 1 if line >= d else line
distance = math.fabs(line - d)
score = int(c.score() - float(decay_factor) / (distance * distance))
c.setScore(score if score > 0 else 0)
result.extend(comments)
result.extend(exceptions)
else:
result.extend(preprocess_comments(ast_f2, []))
return result
def _cleanupPyLintComments(filename, abort):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
old_code = getFileContents(filename)
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def _cleanupPyLintComments(filename, abort):
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, )
old_code = getFileContents(filename)
# Baron does assertions too, and all kinds of strange errors, pylint: disable=broad-except
try:
red = RedBaron(old_code)
except Exception:
if abort:
raise
return
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
with open(filename, "w") as source_code:
source_code.write(red.dumps())
def process_code(py_code):
ast = RedBaron(py_code)
for node in ast.find_all("atomtrailers"):
node.value[0].replace("foo")
return ast.dumps()
def main():
"""Rewrite Thrift-generated Python clients to handle recursive structs. For
more details see: https://issues.apache.org/jira/browse/THRIFT-2642.
Requires package `RedBaron`, available via pip:
$ pip install redbaron
To use:
$ thrift -gen py mapd.thrift
$ mv gen-py/mapd/ttypes.py gen-py/mapd/ttypes-backup.py
$ python fix_recursive_structs.py gen-py/mapd/ttypes-backup.py gen-py/mapd/ttypes.py
"""
in_file = open(sys.argv[1], 'r')
out_file = open(sys.argv[2], 'w')
red_ast = RedBaron(in_file.read())
thrift_specs = [ts.parent for ts in red_ast.find_all(
'name', 'thrift_spec') if ts.parent.type == 'assignment' and ts.parent.parent.name in ['TDatumVal', 'TColumnData']]
nodes = []
for ts in thrift_specs:
node = ts.copy()
node.target = ts.parent.name + '.' + str(node.target)
nodes.append(node)
ts.value = 'None'
red_ast.extend(nodes)
out_file.write(red_ast.dumps())
def autoformat(filename, abort=False):
from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module
ParsingError, # @UnresolvedImport
)
from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module
RedBaron, # @UnresolvedImport
)
my_print("Consider", filename, end=": ")
old_code = open(filename, "r").read()
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
for node in red.find_all("CommentNode"):
try:
_updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
new_code = red.dumps()
if new_code != old_code:
new_name = filename + ".new"
with open(new_name, "w") as source_code:
source_code.write(red.dumps())
if os.name == "nt":
cleanupWindowsNewlines(new_name)
# There is no way to safely replace a file on Windows, but lets try on Linux
# at least.
old_stat = os.stat(filename)
try:
os.rename(new_name, filename)
except OSError:
shutil.copyfile(new_name, filename)
os.unlink(new_name)
os.chmod(filename, old_stat.st_mode)
my_print("updated.")
changed = 1
else:
my_print("OK.")
changed = 0
return changed
def main():
with open("testcode.py", "r") as source_code:
red = RedBaron(source_code.read())
# print(red.help()) ## try this to see the data structure of 'red'
defNodes = red.find_all('DefNode')
# print(defNode.help()) ## try this to see the data structure of 'defNode'
for defNode in defNodes:
print(defNode.help())
def _refactor_content(content, info, request):
red = RedBaron(content)
for func in red.find_all('def'):
for decorator in func.decorators:
steps = re.findall(r'[\'"](.*?)[\'"]', decorator.call.__str__())
if len(steps) > 0 and steps[0] == info.step_text:
_refactor_impl(decorator, func, request)
return red.dumps()
def test_node_next_recursive():
red = RedBaron("def a():\n b = 1\n c = 1\ndef c():\n d = 1")
first_def, second_def = red.find_all('def')
assert first_def.next is second_def
assert second_def.next_recursive is None
assert first_def[0].next_recursive == first_def[1]
assert first_def[1].next_recursive == second_def
def test_node_previous_recursive():
red = RedBaron("def a():\n b = 1\ndef c():\n d = 1\n e = 1\n")
first_def, second_def = red.find_all('def')
assert first_def.previous_recursive is None
assert second_def.previous_recursive is first_def
assert second_def[0].previous_recursive == first_def
assert second_def[1].previous_recursive == second_def[0]
def load_steps(content, file_name):
try:
red = RedBaron(content)
for func in red.find_all('def'):
for decorator in func.decorators:
if decorator.value.__str__() == 'step':
steps = re.findall(r'[\'"](.*?)[\'"]',
decorator.call.__str__())
add_steps(file_name, func, steps)
except Exception:
logging.error("Failed to parse {}.".format(file_name))
def modify(self, base, additional):
t0 = RedBaron(base)
t1 = RedBaron(additional)
d = OrderedDict()
used = set()
for node in t0.find_all("def", recursive=False):
used.add(node.name)
d[node.name] = node
for node in t1.find_all("def", recursive=False):
if node.name in used:
original = d[node.name]
self.on_update(node.name, node, original)
else:
newnode = self.on_create(node.name, node)
if newnode is not None:
t0.append(newnode)
d[node.name] = newnode
self.on_additional(t0, d)
return t0
def test_node_previous_recursive():
red = RedBaron("def a():\n b = 1\ndef c():\n d = 1")
assert red.previous is None
assert red.previous_recursive is None
first, second = red.find_all('def')
assert second.previous is first
inner = second.value.node_list
assert inner[2].previous == inner[1]
assert inner[2].previous_recursive == inner[1]
assert inner[1].previous == inner[0]
assert inner[1].previous_recursive == inner[0]
assert inner[0].previous == None
assert inner[0].previous_recursive == first
def test_node_next_recursive():
red = RedBaron("def a():\n b = 1\ndef c():\n d = 1")
assert red[1].next is None
assert red[1].next_recursive is None
first, second = red.find_all('def')
assert first.next is second
inner = first.value.node_list
assert inner[0].next == inner[1]
assert inner[0].next_recursive == inner[1]
assert inner[1].next == inner[2]
assert inner[1].next_recursive == inner[2]
assert inner[2].next == None
assert inner[2].next_recursive == second
def test_node_next_recursive():
red = RedBaron("def a():\n b = 1\ndef c():\n d = 1")
assert red.next is None
assert red.next_recursive is None
first, second = red.find_all('def')
assert first.next is second
inner = first.value.node_list
assert inner[0].next == inner[1]
assert inner[0].next_recursive == inner[1]
assert inner[1].next == inner[2]
assert inner[1].next_recursive == inner[2]
assert inner[2].next == None
assert inner[2].next_recursive == second
def test_node_previous_recursive():
red = RedBaron("def a():\n b = 1\ndef c():\n d = 1")
assert red[0].previous is None
assert red[0].previous_recursive is None
first, second = red.find_all('def')
assert second.previous is first
inner = second.value.node_list
assert inner[2].previous == inner[1]
assert inner[2].previous_recursive == inner[1]
assert inner[1].previous == inner[0]
assert inner[1].previous_recursive == inner[0]
assert inner[0].previous == None
assert inner[0].previous_recursive == first
def compare(s1, s2, decay_factor = PYTHON_DECAY_FACTOR):
try:
red1 = RedBaron(s1)
red2 = RedBaron(s2)
result = []
defs = red2.find_all('def')
length = len(defs)
for ast_f2 in defs:
ast_f1 = red1.find('def', name = ast_f2.name)
if ast_f1 is not None:
additions, deletions = preprocess_files(ast_f1.dumps(),
ast_f2.dumps(),
ast_f2.absolute_bounding_box.top_left.line)
comments, exceptions = preprocess_comments(ast_f2, additions)
for a in additions:
for c in comments:
line, _ = c.left_bounds
distance = line - a
score = compute_addition(c, distance, decay_factor)
c.setScore(score if score > 0 else 0)
for d in deletions:
for c in comments:
line, _ = c.left_bounds
line = line + 1 if line >= d else line
distance = line - d
score = compute_deletion(c, distance, decay_factor)
c.setScore(score if score > 0 else 0)
result.extend(comments)
result.extend(exceptions)
else:
comments, _ = preprocess_comments(ast_f2, [])
result.extend(comments)
print_to_log('Result: ' + str(result))
return result
except Exception as e:
err = 'CommentHealth compare error: ' + str(e)
return []
import optparse
import json
parser = optparse.OptionParser()
parser.add_option('-s', '--source', dest="source_file")
options, remainder = parser.parse_args()
if options.source_file:
source_code = open(options.source_file, "r")
else:
source_code = sys.stdin
imported_modules = []
red = RedBaron(source_code.read())
import_nodes = red.find_all(lambda identifier: identifier == "import" or identifier == "from_import")
for import_node in import_nodes:
lineno = import_node.absolute_bounding_box.top_left.line
code = import_node.dumps()
modules = None
def is_valid_module(module_path):
return not (module_path.endswith(".(") or module_path.endswith(".)"))
if import_node.type == "from_import":
modules = filter(is_valid_module, import_node.full_path_modules())
elif import_node.type == "import":
modules = import_node.modules()
imported_modules.append({
import copy
import sys
from redbaron import RedBaron
in_file = out_file = sys.argv[1]
code = open(in_file).read()
red_baron = RedBaron(code)
super_nodes = red_baron.find_all('AtomtrailersNode')
for super_node in super_nodes:
node = copy.copy(super_node)
class_name = node.find_all('name')[1].name.dumps()
while node.parent:
node = node.parent
if node.name == class_name:
super_node.value[1] = '()'
with open(out_file, "w") as fh:
fh.write(red_baron.dumps())
def comment_assignments_in_file(filename, assignment_names, dry_run=True, backup_filename=None):
if isinstance(assignment_names, basestring):
assignment_names = [assignment_names]
else:
assignment_names = assignment_names[:]
current_file_data = open(filename).read()
for assignment_name in assignment_names[:]:
if assignment_name in current_file_data:
continue
if assignment_name in assignment_names:
assignment_names.remove(assignment_name)
if not assignment_names:
return ''
replace_lines = {}
rb = RedBaron(current_file_data)
for assignment_node in rb.find_all('assignment'):
for assignment_name in assignment_names:
# Only target direct assignments to a variable.
name_node = assignment_node.find('name', value=assignment_name)
if not name_node:
continue
if assignment_node.target.type != 'name':
continue
# Build a new node that comments out the existing assignment node.
indentation = '{}# '.format(assignment_node.indentation or '')
new_node_content = indent(assignment_node.dumps(), indentation)
new_node_lines = new_node_content.splitlines()
# Add a pass statement in case the assignment block is the only
# child in a parent code block to prevent a syntax error.
if assignment_node.indentation:
new_node_lines[0] = new_node_lines[0].replace(indentation, '{}pass # '.format(assignment_node.indentation or ''), 1)
new_node_lines[0] = '{0}This setting is now configured via the Tower API.\n{1}'.format(indentation, new_node_lines[0])
# Store new node lines in dictionary to be replaced in file.
start_lineno = assignment_node.absolute_bounding_box.top_left.line
end_lineno = assignment_node.absolute_bounding_box.bottom_right.line
for n, new_node_line in enumerate(new_node_lines):
new_lineno = start_lineno + n
assert new_lineno <= end_lineno
replace_lines[new_lineno] = new_node_line
if not replace_lines:
return ''
# Iterate through all lines in current file and replace as needed.
current_file_lines = current_file_data.splitlines()
new_file_lines = []
for n, line in enumerate(current_file_lines):
new_file_lines.append(replace_lines.get(n + 1, line))
new_file_data = '\n'.join(new_file_lines)
new_file_lines = new_file_data.splitlines()
# If changed, syntax check and write the new file; return a diff of changes.
diff_lines = []
if new_file_data != current_file_data:
compile(new_file_data, filename, 'exec')
if backup_filename:
from_file = backup_filename
else:
from_file = '{}.old'.format(filename)
to_file = filename
diff_lines = list(difflib.unified_diff(current_file_lines, new_file_lines, fromfile=from_file, tofile=to_file, lineterm=''))
if not dry_run:
if backup_filename:
shutil.copy2(filename, backup_filename)
with open(filename, 'wb') as fileobj:
fileobj.write(new_file_data)
return '\n'.join(diff_lines)
min = 1
max = 2
result = scipy.optimize.minimize(f,
x0=[1 for _ in range(function.shape[0])],
bounds=[(min, max)
for _ in range(function.shape[0])
],
tol=1e-3,
method='L-BFGS-B')
print(result)
return result.x
red = RedBaron(open('sample_src/test.py').read())
# print(red.dumps())
defs = red.find_all('DefNode')
print(len(defs))
def hist_for(node):
return list(
map(lambda n: str(type(n)).split(".")[-1][:-2],
node.find_all(lambda x: True)))
function_nodes = []
for d in defs:
function_nodes.append(' '.join(hist_for(d)))
df = pd.DataFrame({'nodes': function_nodes})
def test_find_all_comment_nodes():
red = RedBaron("def f():\n #a\n pass\n#b")
assert [x.value for x in red.find_all('comment')] == ['#a', '#b']
import copy
import sys
from redbaron import RedBaron
in_file = out_file = sys.argv[1]
code = open(in_file).read()
red_baron = RedBaron(code)
super_nodes = red_baron.find_all('AtomtrailersNode')
for super_node in super_nodes:
node = copy.copy(super_node)
class_name = node.find_all('name')[1].name.dumps()
while node.parent:
node = node.parent
if node.name == class_name:
super_node.value[1] = '()'
with open(out_file, "w") as fh:
fh.write(red_baron.dumps())
class Parser:
def __init__(self, filename):
self.code = ""
self.message = ""
error_message = ""
error_start_pos = ""
if filename == "ssg":
file_path = Path.cwd() / "ssg.py"
else:
file_path = Path.cwd() / "ssg" / "{}.py".format(filename)
grammar = parso.load_grammar()
module = grammar.parse(path=file_path.resolve())
self.success = len(grammar.iter_errors(module)) == 0
if self.success:
with open(file_path.resolve(), "r") as source_code:
self.code = RedBaron(source_code.read())
else:
error_message = grammar.iter_errors(module)[0].message
error_start_pos = grammar.iter_errors(module)[0].start_pos[0]
self.message = "{} on or around line {} in `{}`.".format(
error_message, error_start_pos, file_path.name
)
def get_by_name(self, type, name, code=None):
if code is None:
item = self.code.find_all(type, lambda node: node.name == name)
else:
item = code.find_all(type, lambda node: node.name == name)
return SourceCode(True, item[0]) if len(item) > 0 else SourceCode(False, [])
def get_call(self, value, code):
call = code.find("call", lambda node: node.previous.value == value)
return SourceCode(True, call) if call is not None and len(call) > 0 else SourceCode(False, [])
def get_args(self, code):
return list(
code.find_all("call_argument").map(
lambda node: str(node.target) + ":" + str(node.value).replace("'", '"')
)
)
def get_by_value(self, type, value, code=None):
if code is None:
item = self.code.find_all(type, lambda node: str(node.target) == value)
else:
item = code.find_all(type, lambda node: str(node.target) == value)
return SourceCode(True, item[0]) if len(item) > 0 else SourceCode(False, [])
def get_imports(self):
imports = []
self.code.find_all(
"import",
lambda node: node.find_all(
"dotted_as_name", lambda node: imports.append(str(node))
),
)
return imports
def get_from_import(self, value):
imports = self.code.find_all(
"from_import",
lambda node: "".join(list(node.value.node_list.map(lambda node: str(node))))
== value,
).find_all("name_as_name")
return list(imports.map(lambda node: node.value))
def flatten(self, dictionary):
def _flatten(node):
trimmed = re.sub(r"\"|'", "", node.key.value)
flattened = []
if node.value.type is "list":
for item in node.value.node_list:
if item.type is not "comma":
flattened.append("{}:{}".format(trimmed, str(item)))
else:
flattened.append("{}:{}".format(trimmed, node.value.value))
return flattened
items = list(dictionary.find_all("dictitem").map(lambda node: _flatten(node)))
return [item for sublist in items for item in sublist]
def get_conditional(self, values, type, nested=False):
def flat(node):
if node.type == "comparison":
return "{}:{}:{}".format(
str(node.first).replace("'", '"'),
str(node.value).replace(" ", ":"),
str(node.second).replace("'", '"'),
)
elif node.type == "unitary_operator":
return "{}:{}".format(
str(node.value), str(node.target).replace("'", '"')
)
nodes = self.code.value if nested else self.code
for value in values:
final_node = nodes.find_all(type).find(
["comparison", "unitary_operator"], lambda node: flat(node) == value
)
if final_node is not None:
return final_node
return None
import copy
import sys
from redbaron import RedBaron
in_file = out_file = sys.argv[1]
code = open(in_file).read()
red_baron = RedBaron(code)
try_nodes = red_baron.find_all('TryNode')
for node in try_nodes:
if node.find('PassNode'):
t_node = copy.copy(super_node)
class_name = node.find_all('name')[1].name.dumps()
while node.parent:
node = node.parent
if node.name == class_name:
super_node.value[1] = '()'
with open(out_file, "w") as fh:
fh.write(red_baron.dumps())
def test_node_finally_previous_intuitive_excepts():
red = RedBaron("try: pass\nexcept: pass\nexcept: pass\nfinally: pass\n")
assert red.find("finally").previous_intuitive is red.find_all("except")[-1]
def test_node_exceptnode_previous_intuitive_except_except():
red = RedBaron("try: pass\nexcept: pass\nexcept: pass")
assert red.find_all("except")[1].previous_intuitive is red.find("except")
def test_node_exceptnode_next_intuitive_except_else():
red = RedBaron("try: pass\nexcept: pass\nexcept: pass\nelse: pass")
assert red.find("except").next_intuitive is red.find_all("except")[1]
from redbaron import RedBaron
from file_utils import listfiles
from file_utils import read_file
from sys import argv
project_path = argv[1]
for file_name in listfiles(project_path, '.py'):
source_code = read_file(file_name)
red = RedBaron(source_code)
resp = red.find_all('ClassNode')
while resp:
if resp[0].inherit_from.find('namenode', value='object'):
print(file_name)
print(resp[0].name)
resp.pop(0)
def autoformat(filename, abort = False):
# All the complexity in one place, pylint: disable=too-many-branches,too-many-locals,too-many-statements
from baron.parser import ParsingError # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module
from redbaron import RedBaron # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module
my_print("Consider", filename, end = ": ")
old_code = open(filename, 'r').read()
try:
red = RedBaron(old_code)
# red = RedBaron(old_code.rstrip()+'\n')
except ParsingError:
if abort:
raise
my_print("PARSING ERROR.")
return 2
def updateCall(call_node):
max_len = 0
for argument in call_node:
if argument.type == "argument_generator_comprehension":
return
if hasattr(argument, "target") and argument.target is not None:
key = argument.target.value
else:
key = None
if key is not None:
max_len = max(max_len, len(key))
if '\n' not in call_node.second_formatting.dumps():
del call_node.second_formatting[:]
del call_node.third_formatting[:]
for argument in call_node:
if hasattr(argument, "target") and argument.target is not None:
key = argument.target.value
else:
key = None
if key is not None:
if not argument.second_formatting:
argument.second_formatting = ' '
if '\n' in str(call_node.second_formatting):
if argument.first_formatting:
spacing = argument.first_formatting[0].value
else:
spacing = ""
if len(key)+len(spacing) != max_len + 1:
argument.first_formatting = ' ' * (max_len - len(key) + 1)
else:
argument.first_formatting = ' '
else:
if '\n' not in str(call_node.second_formatting):
if argument.value.type in ("string", "binary_string", "raw_string"):
argument.value.second_formatting = ""
def updateTuple(tuple_node):
if '\n' not in str(tuple_node.dumps()):
tuple_node.second_formatting = ""
tuple_node.third_formatting = ""
if tuple_node.type == "tuple" and tuple_node.with_parenthesis:
if tuple_node.value.node_list:
if tuple_node.value.node_list[-1].type not in ("yield_atom",):
tuple_node.value.node_list[-1].second_formatting = ""
for argument in tuple_node.value:
if argument.type in ("string", "binary_string", "raw_string"):
argument.second_formatting = ""
def updateString(string_node):
# Skip doc strings for now.
if not hasattr(node.parent, "type") or \
string_node.parent.type in ("class", "def", None):
return
value = string_node.value
def isQuotedWith(quote):
return value.startswith(quote) and value.endswith(quote)
quote = None # For PyLint.
for quote in "'''", '"""', "'", '"':
if isQuotedWith(quote):
break
else:
sys.exit("Error, quote not understood.")
real_value = value[len(quote):-len(quote)]
assert quote + real_value + quote == value
if '\n' not in real_value:
# Single characters, should be quoted with "'"
if len(eval(value)) == 1: # pylint: disable=eval-used
if real_value != "'":
string_node.value = "'" + real_value + "'"
else:
if '"' not in real_value:
string_node.value = '"' + real_value + '"'
def updateDefNode(def_node):
# This is between "def" and function name.
def_node.first_formatting = ' '
# This is after the opening/closing brace, we don't want it there.
def_node.third_formatting = ""
def_node.fourth_formatting = ""
# This is to insert/remove spaces or new lines, depending on line length
# so far, but is not functional at all.
for argument_node in def_node.arguments:
argument_node.first_formatting = ' '
argument_node.second_formatting = ' '
def updateCommentNode(comment_node):
if "pylint:" in str(comment_node.value):
def replacer(part):
def renamer(pylint_token):
# pylint: disable=too-many-return-statements
if pylint_token == "E0602":
return "undefined-variable"
elif pylint_token in ("E0401", "F0401"):
return "import-error"
elif pylint_token == "E1102":
return "not-callable"
elif pylint_token == "E1133":
return " not-an-iterable"
elif pylint_token == "E1128":
return "assignment-from-none"
# Save line length for this until isort is better at long lines.
elif pylint_token == "useless-suppression":
return "I0021"
# elif pylint_token == "I0021":
# return "useless-suppression"
elif pylint_token == "R0911":
return "too-many-return-statements"
elif pylint_token == "R0201":
return "no-self-use"
elif pylint_token == "R0902":
return "too-many-instance-attributes"
elif pylint_token == "R0912":
return "too-many-branches"
elif pylint_token == "R0914":
return "too-many-locals"
elif pylint_token == "R0915":
return "too-many-statements"
elif pylint_token == "W0123":
return "eval-used"
elif pylint_token == "W0603":
return "global-statement"
elif pylint_token == "W0613":
return "unused-argument"
elif pylint_token == "W0622":
return "redefined-builtin"
elif pylint_token == "W0703":
return "broad-except"
else:
return pylint_token
return part.group(1) + ','.join(
sorted(
renamer(token)
for token in
part.group(2).split(',')
)
)
new_value = re.sub(r"(pylint\: disable=)(.*)", replacer, str(comment_node.value), flags = re.M)
comment_node.value = new_value
for node in red.find_all("CallNode"):
try:
updateCall(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("TupleNode"):
try:
updateTuple(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("ListNode"):
try:
updateTuple(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("SetNode"):
try:
updateTuple(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("StringNode"):
try:
updateString(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("DefNode"):
try:
updateDefNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("CommentNode"):
try:
updateCommentNode(node)
except Exception:
my_print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
new_code = red.dumps()
if new_code != old_code:
new_name = filename + ".new"
with open(new_name, 'w') as source_code:
source_code.write(red.dumps())
if os.name == "nt":
cleanupWindowsNewlines(new_name)
# There is no way to safely replace a file on Windows, but lets try on Linux
# at least.
old_stat = os.stat(filename)
try:
os.rename(new_name, filename)
except OSError:
shutil.copy(new_name, filename)
os.unlink(new_name)
os.chmod(filename, old_stat.st_mode)
my_print("updated.")
changed = 1
else:
my_print("OK.")
changed = 0
return changed
def updateDefNode(def_node):
# This is between "def" and function name.
def_node.first_formatting = " "
# This is after the opening/closing brace, we don't want it there.
def_node.third_formatting = ""
def_node.fourth_formatting = ""
# This is to insert/remove spaces or new lines, depending on line length
# so far, but is not functional at all.
for argument_node in def_node.arguments:
argument_node.first_formatting = " "
argument_node.second_formatting = " "
for node in red.find_all("CallNode"):
try:
updateCall(node)
except Exception:
print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
for node in red.find_all("TupleNode"):
try:
updateTuple(node)
except Exception:
print("Problem with", node)
node.help(deep = True, with_formatting = True)
raise
def test_node_trynode_outside_previous_intuitive_except_except():
red = RedBaron("try:\n pass\nexcept: pass\nexcept: pass\nafter")
assert red.find("name",
"after").previous_intuitive is red.find_all("except")[1]
def updateCommentNode(comment_node):
if "pylint:" in str(comment_node.value):
def replacer(part):
return part.group(1) + ','.join(sorted(part.group(2).split(',')))
new_value = re.sub(r"(pylint\: disable=)(.*)",
replacer,
str(comment_node.value),
flags=re.M)
comment_node.value = new_value
for node in red.find_all("CallNode"):
try:
updateCall(node)
except Exception:
print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
for node in red.find_all("TupleNode"):
try:
updateTuple(node)
except Exception:
print("Problem with", node)
node.help(deep=True, with_formatting=True)
raise
def test_node_if_ifelseblock_outside_previous_intuitive_elif_elif():
red = RedBaron("if a:\n pass\nelif a: pass\nelif a: pass\nafter")
assert red.find("name",
"after").previous_intuitive is red.find_all("elif")[-1]
