def build_imported_libraries_vocabulary(script_folder, max_script_count=10000, vocab_size=500, min_count=2): ''' Generates a dictionary of imported library calls to be used as the vocabulary in techniques that utilize bag of words. Args: script_folder (str): Folder location of corpus containing script files max_script_count (int): the maximum number of code scripts to process in the script_folder vocab_size (int): the maximum number of words to be used in the vocabulary (dimension of bag of words vector) min_word_count (int): a word will be included in vocabulary if it appears at least min_count times in the corpus Returns: libraries_ordered_by_count (list): a list of size equal or less than max_vocab_size that contains the most frequent normalized words in the corpus ''' library_count = defaultdict(int) counter = 0 # Retrieve files with script content and process with red baron to identify imported libraries # Altair's JSON format uses the 'content' label for the script code for py_file in sorted(os.listdir(script_folder)): if counter >= max_script_count: break fullpath = os.path.join(script_folder, py_file) with open(fullpath, "r") as py_file_contents: for line in py_file_contents: counter += 1 parsed_json = json.loads(line) try: red = RedBaron(parsed_json['content']) libraries = parse_import_statements( red.find_all("ImportNode")) libraries |= parse_fromimport_statements( red.find_all("FromImportNode")) for library in libraries: library_count[library] += 1 except Exception as e: #logger.info("%s error encountered in %s; skipping file" % (e.__class__.__name__, py_file)) continue if counter >= max_script_count: break # Determine descending order for library based on count libraries_ordered_by_count = [ i[0] for i in sorted( library_count.items(), key=lambda x: (x[1], x[0]), reverse=True) if i[1] > min_count ] # Trim the vocabulary to the requested vocab_size if len(libraries_ordered_by_count) >= vocab_size: libraries_ordered_by_count = libraries_ordered_by_count[:vocab_size] else: logger.warning("Only %d libraries were observed using max_script_count=%d, max_vocab_size=%d and min_word_count=%d" % \ (len(libraries_ordered_by_count), max_script_count, vocab_size, min_count)) return libraries_ordered_by_count
def _extract_libraries(self, document): try: red = RedBaron(document) libraries = parse_import_statements(red.find_all("ImportNode")) libraries |= parse_fromimport_statements(red.find_all("FromImportNode")) # TODO: Skip document and remove from raw list if red baron fails parsing except: libraries = set() logger.info("Red baron error;continuing") return " ".join(libraries)
def functionalize(src): red = RedBaron(src) red.insert(0, 'import pync') for func in red.find_all('def'): func.decorators.append('@pync.curry') for l in red.find_all('list') + red.find_all('list_comprehension'): l.replace("pync.list(%s)" % l) return red.dumps()
def load_real_code(code): # dataset_folder = '/home/smartscript/smartscript_web/static/py_checker/misc/upload' methods = [] codeNames = [] funcNames = [] # for code_file in glob.glob(os.path.join(dataset_folder, '*.py'), recursive=False): # print(code_file) # with open(code_file, 'r') as reader: # code = reader.read() red = RedBaron(code) method_reds = red.find_all('def') for method_red in method_reds: funcName = str(method_red).split(':')[0][4:] methods.append(method_red.dumps()) codeNames.append("uploaded source code") funcNames.append(funcName) # methods1 = [] # codeNames1 = [] # funcNames1 = [] # for m in methods: # methods1.append(m) # for c in codeNames: # codeNames1.append(c) # for f in funcNames: # funcNames1.append(f) # for m in methods: # methods1.append(m) # for c in codeNames: # codeNames1.append(c) # for f in funcNames: # funcNames1.append(f) return methods,codeNames,funcNames
def _cleanupPyLintComments(filename, abort): from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module ParsingError, # @UnresolvedImport ) from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module RedBaron, # @UnresolvedImport ) old_code = getFileContents(filename) try: red = RedBaron(old_code) # red = RedBaron(old_code.rstrip()+'\n') except ParsingError: if abort: raise my_print("PARSING ERROR.") return 2 for node in red.find_all("CommentNode"): try: _updateCommentNode(node) except Exception: my_print("Problem with", node) node.help(deep=True, with_formatting=True) raise new_code = red.dumps() if new_code != old_code: with open(filename, "w") as source_code: source_code.write(red.dumps())
def main() -> Optional[int]: has_errors = False for model_file_path in get_input_models_files(): with open(model_file_path) as f: baron_tree = RedBaron(f.read()) for assignment in baron_tree.find_all('assignment'): nodes_to_check = [ assignment.find('comment'), ] if isinstance(assignment.next, CommentNode): nodes_to_check.append(assignment.next) assignment_str = assignment.dumps() is_nullable_field = ( NULL_TRUE_KWARG_RE.search(assignment_str) or 'NullBooleanField' in assignment_str ) if is_nullable_field and not any(filter(has_valid_comment, nodes_to_check)): has_errors = True field_name = assignment.target print( # noqa: T001 '{0}:{1} Field "{2}" needs a valid comment for its "null=True"'.format( model_file_path, get_node_line(assignment), field_name, ), ) if has_errors: return 1
def compare(s1, s2, decay_factor = DEFAULT_DECAY_FACTOR): red1 = RedBaron(s1) red2 = RedBaron(s2) result = [] for ast_f2 in red2.find_all('def'): ast_f1 = red1.find('def', name = ast_f2.name) if ast_f1 is not None: additions, deletions = preprocess_files(ast_f1.dumps(), ast_f2.dumps()) comments, exceptions = preprocess_comments(ast_f2, additions) for a in additions: for c in comments: line, _ = c.left_bounds distance = math.fabs(line - a) score = int(c.score() - float(decay_factor) / (distance * distance)) c.setScore(score if score > 0 else 0) for d in deletions: for c in comments: line, _ = c.left_bounds line = line + 1 if line >= d else line distance = math.fabs(line - d) score = int(c.score() - float(decay_factor) / (distance * distance)) c.setScore(score if score > 0 else 0) result.extend(comments) result.extend(exceptions) else: result.extend(preprocess_comments(ast_f2, [])) return result
def _cleanupPyLintComments(filename, abort): from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module RedBaron, ) old_code = getFileContents(filename) # Baron does assertions too, and all kinds of strange errors, pylint: disable=broad-except try: red = RedBaron(old_code) except Exception: if abort: raise return for node in red.find_all("CommentNode"): try: _updateCommentNode(node) except Exception: my_print("Problem with", node) node.help(deep=True, with_formatting=True) raise new_code = red.dumps() if new_code != old_code: with open(filename, "w") as source_code: source_code.write(red.dumps())
def process_code(py_code): ast = RedBaron(py_code) for node in ast.find_all("atomtrailers"): node.value[0].replace("foo") return ast.dumps()
def main(): """Rewrite Thrift-generated Python clients to handle recursive structs. For more details see: https://issues.apache.org/jira/browse/THRIFT-2642. Requires package `RedBaron`, available via pip: $ pip install redbaron To use: $ thrift -gen py mapd.thrift $ mv gen-py/mapd/ttypes.py gen-py/mapd/ttypes-backup.py $ python fix_recursive_structs.py gen-py/mapd/ttypes-backup.py gen-py/mapd/ttypes.py """ in_file = open(sys.argv[1], 'r') out_file = open(sys.argv[2], 'w') red_ast = RedBaron(in_file.read()) thrift_specs = [ts.parent for ts in red_ast.find_all( 'name', 'thrift_spec') if ts.parent.type == 'assignment' and ts.parent.parent.name in ['TDatumVal', 'TColumnData']] nodes = [] for ts in thrift_specs: node = ts.copy() node.target = ts.parent.name + '.' + str(node.target) nodes.append(node) ts.value = 'None' red_ast.extend(nodes) out_file.write(red_ast.dumps())
def autoformat(filename, abort=False): from baron.parser import ( # pylint: disable=I0021,import-error,no-name-in-module ParsingError, # @UnresolvedImport ) from redbaron import ( # pylint: disable=I0021,import-error,no-name-in-module RedBaron, # @UnresolvedImport ) my_print("Consider", filename, end=": ") old_code = open(filename, "r").read() try: red = RedBaron(old_code) # red = RedBaron(old_code.rstrip()+'\n') except ParsingError: if abort: raise my_print("PARSING ERROR.") return 2 for node in red.find_all("CommentNode"): try: _updateCommentNode(node) except Exception: my_print("Problem with", node) node.help(deep=True, with_formatting=True) raise new_code = red.dumps() if new_code != old_code: new_name = filename + ".new" with open(new_name, "w") as source_code: source_code.write(red.dumps()) if os.name == "nt": cleanupWindowsNewlines(new_name) # There is no way to safely replace a file on Windows, but lets try on Linux # at least. old_stat = os.stat(filename) try: os.rename(new_name, filename) except OSError: shutil.copyfile(new_name, filename) os.unlink(new_name) os.chmod(filename, old_stat.st_mode) my_print("updated.") changed = 1 else: my_print("OK.") changed = 0 return changed
def main(): with open("testcode.py", "r") as source_code: red = RedBaron(source_code.read()) # print(red.help()) ## try this to see the data structure of 'red' defNodes = red.find_all('DefNode') # print(defNode.help()) ## try this to see the data structure of 'defNode' for defNode in defNodes: print(defNode.help())
def _refactor_content(content, info, request): red = RedBaron(content) for func in red.find_all('def'): for decorator in func.decorators: steps = re.findall(r'[\'"](.*?)[\'"]', decorator.call.__str__()) if len(steps) > 0 and steps[0] == info.step_text: _refactor_impl(decorator, func, request) return red.dumps()
def test_node_next_recursive(): red = RedBaron("def a():\n b = 1\n c = 1\ndef c():\n d = 1") first_def, second_def = red.find_all('def') assert first_def.next is second_def assert second_def.next_recursive is None assert first_def[0].next_recursive == first_def[1] assert first_def[1].next_recursive == second_def
def test_node_previous_recursive(): red = RedBaron("def a():\n b = 1\ndef c():\n d = 1\n e = 1\n") first_def, second_def = red.find_all('def') assert first_def.previous_recursive is None assert second_def.previous_recursive is first_def assert second_def[0].previous_recursive == first_def assert second_def[1].previous_recursive == second_def[0]
def load_steps(content, file_name): try: red = RedBaron(content) for func in red.find_all('def'): for decorator in func.decorators: if decorator.value.__str__() == 'step': steps = re.findall(r'[\'"](.*?)[\'"]', decorator.call.__str__()) add_steps(file_name, func, steps) except Exception: logging.error("Failed to parse {}.".format(file_name))
def modify(self, base, additional): t0 = RedBaron(base) t1 = RedBaron(additional) d = OrderedDict() used = set() for node in t0.find_all("def", recursive=False): used.add(node.name) d[node.name] = node for node in t1.find_all("def", recursive=False): if node.name in used: original = d[node.name] self.on_update(node.name, node, original) else: newnode = self.on_create(node.name, node) if newnode is not None: t0.append(newnode) d[node.name] = newnode self.on_additional(t0, d) return t0
def test_node_previous_recursive(): red = RedBaron("def a():\n b = 1\ndef c():\n d = 1") assert red.previous is None assert red.previous_recursive is None first, second = red.find_all('def') assert second.previous is first inner = second.value.node_list assert inner[2].previous == inner[1] assert inner[2].previous_recursive == inner[1] assert inner[1].previous == inner[0] assert inner[1].previous_recursive == inner[0] assert inner[0].previous == None assert inner[0].previous_recursive == first
def test_node_next_recursive(): red = RedBaron("def a():\n b = 1\ndef c():\n d = 1") assert red[1].next is None assert red[1].next_recursive is None first, second = red.find_all('def') assert first.next is second inner = first.value.node_list assert inner[0].next == inner[1] assert inner[0].next_recursive == inner[1] assert inner[1].next == inner[2] assert inner[1].next_recursive == inner[2] assert inner[2].next == None assert inner[2].next_recursive == second
def test_node_next_recursive(): red = RedBaron("def a():\n b = 1\ndef c():\n d = 1") assert red.next is None assert red.next_recursive is None first, second = red.find_all('def') assert first.next is second inner = first.value.node_list assert inner[0].next == inner[1] assert inner[0].next_recursive == inner[1] assert inner[1].next == inner[2] assert inner[1].next_recursive == inner[2] assert inner[2].next == None assert inner[2].next_recursive == second
def test_node_previous_recursive(): red = RedBaron("def a():\n b = 1\ndef c():\n d = 1") assert red[0].previous is None assert red[0].previous_recursive is None first, second = red.find_all('def') assert second.previous is first inner = second.value.node_list assert inner[2].previous == inner[1] assert inner[2].previous_recursive == inner[1] assert inner[1].previous == inner[0] assert inner[1].previous_recursive == inner[0] assert inner[0].previous == None assert inner[0].previous_recursive == first
def compare(s1, s2, decay_factor = PYTHON_DECAY_FACTOR): try: red1 = RedBaron(s1) red2 = RedBaron(s2) result = [] defs = red2.find_all('def') length = len(defs) for ast_f2 in defs: ast_f1 = red1.find('def', name = ast_f2.name) if ast_f1 is not None: additions, deletions = preprocess_files(ast_f1.dumps(), ast_f2.dumps(), ast_f2.absolute_bounding_box.top_left.line) comments, exceptions = preprocess_comments(ast_f2, additions) for a in additions: for c in comments: line, _ = c.left_bounds distance = line - a score = compute_addition(c, distance, decay_factor) c.setScore(score if score > 0 else 0) for d in deletions: for c in comments: line, _ = c.left_bounds line = line + 1 if line >= d else line distance = line - d score = compute_deletion(c, distance, decay_factor) c.setScore(score if score > 0 else 0) result.extend(comments) result.extend(exceptions) else: comments, _ = preprocess_comments(ast_f2, []) result.extend(comments) print_to_log('Result: ' + str(result)) return result except Exception as e: err = 'CommentHealth compare error: ' + str(e) return []
import optparse import json parser = optparse.OptionParser() parser.add_option('-s', '--source', dest="source_file") options, remainder = parser.parse_args() if options.source_file: source_code = open(options.source_file, "r") else: source_code = sys.stdin imported_modules = [] red = RedBaron(source_code.read()) import_nodes = red.find_all(lambda identifier: identifier == "import" or identifier == "from_import") for import_node in import_nodes: lineno = import_node.absolute_bounding_box.top_left.line code = import_node.dumps() modules = None def is_valid_module(module_path): return not (module_path.endswith(".(") or module_path.endswith(".)")) if import_node.type == "from_import": modules = filter(is_valid_module, import_node.full_path_modules()) elif import_node.type == "import": modules = import_node.modules() imported_modules.append({
import copy import sys from redbaron import RedBaron in_file = out_file = sys.argv[1] code = open(in_file).read() red_baron = RedBaron(code) super_nodes = red_baron.find_all('AtomtrailersNode') for super_node in super_nodes: node = copy.copy(super_node) class_name = node.find_all('name')[1].name.dumps() while node.parent: node = node.parent if node.name == class_name: super_node.value[1] = '()' with open(out_file, "w") as fh: fh.write(red_baron.dumps())
def comment_assignments_in_file(filename, assignment_names, dry_run=True, backup_filename=None): if isinstance(assignment_names, basestring): assignment_names = [assignment_names] else: assignment_names = assignment_names[:] current_file_data = open(filename).read() for assignment_name in assignment_names[:]: if assignment_name in current_file_data: continue if assignment_name in assignment_names: assignment_names.remove(assignment_name) if not assignment_names: return '' replace_lines = {} rb = RedBaron(current_file_data) for assignment_node in rb.find_all('assignment'): for assignment_name in assignment_names: # Only target direct assignments to a variable. name_node = assignment_node.find('name', value=assignment_name) if not name_node: continue if assignment_node.target.type != 'name': continue # Build a new node that comments out the existing assignment node. indentation = '{}# '.format(assignment_node.indentation or '') new_node_content = indent(assignment_node.dumps(), indentation) new_node_lines = new_node_content.splitlines() # Add a pass statement in case the assignment block is the only # child in a parent code block to prevent a syntax error. if assignment_node.indentation: new_node_lines[0] = new_node_lines[0].replace(indentation, '{}pass # '.format(assignment_node.indentation or ''), 1) new_node_lines[0] = '{0}This setting is now configured via the Tower API.\n{1}'.format(indentation, new_node_lines[0]) # Store new node lines in dictionary to be replaced in file. start_lineno = assignment_node.absolute_bounding_box.top_left.line end_lineno = assignment_node.absolute_bounding_box.bottom_right.line for n, new_node_line in enumerate(new_node_lines): new_lineno = start_lineno + n assert new_lineno <= end_lineno replace_lines[new_lineno] = new_node_line if not replace_lines: return '' # Iterate through all lines in current file and replace as needed. current_file_lines = current_file_data.splitlines() new_file_lines = [] for n, line in enumerate(current_file_lines): new_file_lines.append(replace_lines.get(n + 1, line)) new_file_data = '\n'.join(new_file_lines) new_file_lines = new_file_data.splitlines() # If changed, syntax check and write the new file; return a diff of changes. diff_lines = [] if new_file_data != current_file_data: compile(new_file_data, filename, 'exec') if backup_filename: from_file = backup_filename else: from_file = '{}.old'.format(filename) to_file = filename diff_lines = list(difflib.unified_diff(current_file_lines, new_file_lines, fromfile=from_file, tofile=to_file, lineterm='')) if not dry_run: if backup_filename: shutil.copy2(filename, backup_filename) with open(filename, 'wb') as fileobj: fileobj.write(new_file_data) return '\n'.join(diff_lines)
min = 1 max = 2 result = scipy.optimize.minimize(f, x0=[1 for _ in range(function.shape[0])], bounds=[(min, max) for _ in range(function.shape[0]) ], tol=1e-3, method='L-BFGS-B') print(result) return result.x red = RedBaron(open('sample_src/test.py').read()) # print(red.dumps()) defs = red.find_all('DefNode') print(len(defs)) def hist_for(node): return list( map(lambda n: str(type(n)).split(".")[-1][:-2], node.find_all(lambda x: True))) function_nodes = [] for d in defs: function_nodes.append(' '.join(hist_for(d))) df = pd.DataFrame({'nodes': function_nodes})
def test_find_all_comment_nodes(): red = RedBaron("def f():\n #a\n pass\n#b") assert [x.value for x in red.find_all('comment')] == ['#a', '#b']
class Parser: def __init__(self, filename): self.code = "" self.message = "" error_message = "" error_start_pos = "" if filename == "ssg": file_path = Path.cwd() / "ssg.py" else: file_path = Path.cwd() / "ssg" / "{}.py".format(filename) grammar = parso.load_grammar() module = grammar.parse(path=file_path.resolve()) self.success = len(grammar.iter_errors(module)) == 0 if self.success: with open(file_path.resolve(), "r") as source_code: self.code = RedBaron(source_code.read()) else: error_message = grammar.iter_errors(module)[0].message error_start_pos = grammar.iter_errors(module)[0].start_pos[0] self.message = "{} on or around line {} in `{}`.".format( error_message, error_start_pos, file_path.name ) def get_by_name(self, type, name, code=None): if code is None: item = self.code.find_all(type, lambda node: node.name == name) else: item = code.find_all(type, lambda node: node.name == name) return SourceCode(True, item[0]) if len(item) > 0 else SourceCode(False, []) def get_call(self, value, code): call = code.find("call", lambda node: node.previous.value == value) return SourceCode(True, call) if call is not None and len(call) > 0 else SourceCode(False, []) def get_args(self, code): return list( code.find_all("call_argument").map( lambda node: str(node.target) + ":" + str(node.value).replace("'", '"') ) ) def get_by_value(self, type, value, code=None): if code is None: item = self.code.find_all(type, lambda node: str(node.target) == value) else: item = code.find_all(type, lambda node: str(node.target) == value) return SourceCode(True, item[0]) if len(item) > 0 else SourceCode(False, []) def get_imports(self): imports = [] self.code.find_all( "import", lambda node: node.find_all( "dotted_as_name", lambda node: imports.append(str(node)) ), ) return imports def get_from_import(self, value): imports = self.code.find_all( "from_import", lambda node: "".join(list(node.value.node_list.map(lambda node: str(node)))) == value, ).find_all("name_as_name") return list(imports.map(lambda node: node.value)) def flatten(self, dictionary): def _flatten(node): trimmed = re.sub(r"\"|'", "", node.key.value) flattened = [] if node.value.type is "list": for item in node.value.node_list: if item.type is not "comma": flattened.append("{}:{}".format(trimmed, str(item))) else: flattened.append("{}:{}".format(trimmed, node.value.value)) return flattened items = list(dictionary.find_all("dictitem").map(lambda node: _flatten(node))) return [item for sublist in items for item in sublist] def get_conditional(self, values, type, nested=False): def flat(node): if node.type == "comparison": return "{}:{}:{}".format( str(node.first).replace("'", '"'), str(node.value).replace(" ", ":"), str(node.second).replace("'", '"'), ) elif node.type == "unitary_operator": return "{}:{}".format( str(node.value), str(node.target).replace("'", '"') ) nodes = self.code.value if nested else self.code for value in values: final_node = nodes.find_all(type).find( ["comparison", "unitary_operator"], lambda node: flat(node) == value ) if final_node is not None: return final_node return None
import copy import sys from redbaron import RedBaron in_file = out_file = sys.argv[1] code = open(in_file).read() red_baron = RedBaron(code) try_nodes = red_baron.find_all('TryNode') for node in try_nodes: if node.find('PassNode'): t_node = copy.copy(super_node) class_name = node.find_all('name')[1].name.dumps() while node.parent: node = node.parent if node.name == class_name: super_node.value[1] = '()' with open(out_file, "w") as fh: fh.write(red_baron.dumps())
def test_node_finally_previous_intuitive_excepts(): red = RedBaron("try: pass\nexcept: pass\nexcept: pass\nfinally: pass\n") assert red.find("finally").previous_intuitive is red.find_all("except")[-1]
def test_node_exceptnode_previous_intuitive_except_except(): red = RedBaron("try: pass\nexcept: pass\nexcept: pass") assert red.find_all("except")[1].previous_intuitive is red.find("except")
def test_node_exceptnode_next_intuitive_except_else(): red = RedBaron("try: pass\nexcept: pass\nexcept: pass\nelse: pass") assert red.find("except").next_intuitive is red.find_all("except")[1]
from redbaron import RedBaron from file_utils import listfiles from file_utils import read_file from sys import argv project_path = argv[1] for file_name in listfiles(project_path, '.py'): source_code = read_file(file_name) red = RedBaron(source_code) resp = red.find_all('ClassNode') while resp: if resp[0].inherit_from.find('namenode', value='object'): print(file_name) print(resp[0].name) resp.pop(0)
def autoformat(filename, abort = False): # All the complexity in one place, pylint: disable=too-many-branches,too-many-locals,too-many-statements from baron.parser import ParsingError # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module from redbaron import RedBaron # @UnresolvedImport pylint: disable=I0021,import-error,no-name-in-module my_print("Consider", filename, end = ": ") old_code = open(filename, 'r').read() try: red = RedBaron(old_code) # red = RedBaron(old_code.rstrip()+'\n') except ParsingError: if abort: raise my_print("PARSING ERROR.") return 2 def updateCall(call_node): max_len = 0 for argument in call_node: if argument.type == "argument_generator_comprehension": return if hasattr(argument, "target") and argument.target is not None: key = argument.target.value else: key = None if key is not None: max_len = max(max_len, len(key)) if '\n' not in call_node.second_formatting.dumps(): del call_node.second_formatting[:] del call_node.third_formatting[:] for argument in call_node: if hasattr(argument, "target") and argument.target is not None: key = argument.target.value else: key = None if key is not None: if not argument.second_formatting: argument.second_formatting = ' ' if '\n' in str(call_node.second_formatting): if argument.first_formatting: spacing = argument.first_formatting[0].value else: spacing = "" if len(key)+len(spacing) != max_len + 1: argument.first_formatting = ' ' * (max_len - len(key) + 1) else: argument.first_formatting = ' ' else: if '\n' not in str(call_node.second_formatting): if argument.value.type in ("string", "binary_string", "raw_string"): argument.value.second_formatting = "" def updateTuple(tuple_node): if '\n' not in str(tuple_node.dumps()): tuple_node.second_formatting = "" tuple_node.third_formatting = "" if tuple_node.type == "tuple" and tuple_node.with_parenthesis: if tuple_node.value.node_list: if tuple_node.value.node_list[-1].type not in ("yield_atom",): tuple_node.value.node_list[-1].second_formatting = "" for argument in tuple_node.value: if argument.type in ("string", "binary_string", "raw_string"): argument.second_formatting = "" def updateString(string_node): # Skip doc strings for now. if not hasattr(node.parent, "type") or \ string_node.parent.type in ("class", "def", None): return value = string_node.value def isQuotedWith(quote): return value.startswith(quote) and value.endswith(quote) quote = None # For PyLint. for quote in "'''", '"""', "'", '"': if isQuotedWith(quote): break else: sys.exit("Error, quote not understood.") real_value = value[len(quote):-len(quote)] assert quote + real_value + quote == value if '\n' not in real_value: # Single characters, should be quoted with "'" if len(eval(value)) == 1: # pylint: disable=eval-used if real_value != "'": string_node.value = "'" + real_value + "'" else: if '"' not in real_value: string_node.value = '"' + real_value + '"' def updateDefNode(def_node): # This is between "def" and function name. def_node.first_formatting = ' ' # This is after the opening/closing brace, we don't want it there. def_node.third_formatting = "" def_node.fourth_formatting = "" # This is to insert/remove spaces or new lines, depending on line length # so far, but is not functional at all. for argument_node in def_node.arguments: argument_node.first_formatting = ' ' argument_node.second_formatting = ' ' def updateCommentNode(comment_node): if "pylint:" in str(comment_node.value): def replacer(part): def renamer(pylint_token): # pylint: disable=too-many-return-statements if pylint_token == "E0602": return "undefined-variable" elif pylint_token in ("E0401", "F0401"): return "import-error" elif pylint_token == "E1102": return "not-callable" elif pylint_token == "E1133": return " not-an-iterable" elif pylint_token == "E1128": return "assignment-from-none" # Save line length for this until isort is better at long lines. elif pylint_token == "useless-suppression": return "I0021" # elif pylint_token == "I0021": # return "useless-suppression" elif pylint_token == "R0911": return "too-many-return-statements" elif pylint_token == "R0201": return "no-self-use" elif pylint_token == "R0902": return "too-many-instance-attributes" elif pylint_token == "R0912": return "too-many-branches" elif pylint_token == "R0914": return "too-many-locals" elif pylint_token == "R0915": return "too-many-statements" elif pylint_token == "W0123": return "eval-used" elif pylint_token == "W0603": return "global-statement" elif pylint_token == "W0613": return "unused-argument" elif pylint_token == "W0622": return "redefined-builtin" elif pylint_token == "W0703": return "broad-except" else: return pylint_token return part.group(1) + ','.join( sorted( renamer(token) for token in part.group(2).split(',') ) ) new_value = re.sub(r"(pylint\: disable=)(.*)", replacer, str(comment_node.value), flags = re.M) comment_node.value = new_value for node in red.find_all("CallNode"): try: updateCall(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("TupleNode"): try: updateTuple(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("ListNode"): try: updateTuple(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("SetNode"): try: updateTuple(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("StringNode"): try: updateString(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("DefNode"): try: updateDefNode(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("CommentNode"): try: updateCommentNode(node) except Exception: my_print("Problem with", node) node.help(deep = True, with_formatting = True) raise new_code = red.dumps() if new_code != old_code: new_name = filename + ".new" with open(new_name, 'w') as source_code: source_code.write(red.dumps()) if os.name == "nt": cleanupWindowsNewlines(new_name) # There is no way to safely replace a file on Windows, but lets try on Linux # at least. old_stat = os.stat(filename) try: os.rename(new_name, filename) except OSError: shutil.copy(new_name, filename) os.unlink(new_name) os.chmod(filename, old_stat.st_mode) my_print("updated.") changed = 1 else: my_print("OK.") changed = 0 return changed
def updateDefNode(def_node): # This is between "def" and function name. def_node.first_formatting = " " # This is after the opening/closing brace, we don't want it there. def_node.third_formatting = "" def_node.fourth_formatting = "" # This is to insert/remove spaces or new lines, depending on line length # so far, but is not functional at all. for argument_node in def_node.arguments: argument_node.first_formatting = " " argument_node.second_formatting = " " for node in red.find_all("CallNode"): try: updateCall(node) except Exception: print("Problem with", node) node.help(deep = True, with_formatting = True) raise for node in red.find_all("TupleNode"): try: updateTuple(node) except Exception: print("Problem with", node) node.help(deep = True, with_formatting = True) raise
def test_node_trynode_outside_previous_intuitive_except_except(): red = RedBaron("try:\n pass\nexcept: pass\nexcept: pass\nafter") assert red.find("name", "after").previous_intuitive is red.find_all("except")[1]
def updateCommentNode(comment_node): if "pylint:" in str(comment_node.value): def replacer(part): return part.group(1) + ','.join(sorted(part.group(2).split(','))) new_value = re.sub(r"(pylint\: disable=)(.*)", replacer, str(comment_node.value), flags=re.M) comment_node.value = new_value for node in red.find_all("CallNode"): try: updateCall(node) except Exception: print("Problem with", node) node.help(deep=True, with_formatting=True) raise for node in red.find_all("TupleNode"): try: updateTuple(node) except Exception: print("Problem with", node) node.help(deep=True, with_formatting=True) raise
def test_node_if_ifelseblock_outside_previous_intuitive_elif_elif(): red = RedBaron("if a:\n pass\nelif a: pass\nelif a: pass\nafter") assert red.find("name", "after").previous_intuitive is red.find_all("elif")[-1]